Eats characters until on from follow is found and Lexer state
is balanced at the moment
341114
private void consumeUntilBalanced(IntervalSet follow) {

        log.debug("Lexer entered consumeUntilBalanced with {} and follow {}",
                ls, follow);

        int c;
        do {
            c = input.LA(1);
            // change apostrophe state
            if (c == '\'' && ls.quotOpen == false) {
                ls.aposOpen = !ls.aposOpen;
            }
            // change quot state
            else if (c == '"' && ls.aposOpen == false) {
                ls.quotOpen = !ls.quotOpen;
            } else if (c == '(') {
                ls.parenNest++;
            } else if (c == ')' && ls.parenNest > 0) {
                ls.parenNest--;
            } else if (c == '{') {
                ls.curlyNest++;
            } else if (c == '}' && ls.curlyNest > 0) {
                ls.curlyNest--;
            }
            // handle end of line in string
            else if (c == '\n') {
                if (ls.quotOpen) ls.quotOpen = false;
                else if (ls.aposOpen) ls.aposOpen = false;
            } else if (c == Token.EOF) {
                log.info("Unexpected EOF during consumeUntilBalanced, EOF not consumed");
                return;
            }

            input.consume();
            // log result
            if (log.isTraceEnabled())
                log.trace("Lexer consumes '{}'({}) until balanced ({}).",
                        new Object[]{
                                Character.toString((char) c),
                                Integer.toString(c),
                                ls});

        } while (!(ls.isBalanced() && follow.contains(c)));
    }
362841
def complete_watch(self, text, *_):
        
        return [t + " " for t in self.engine.cached_descriptions if t.startswith(text)]
91927
function tokenize (str) {
  let pos = 0
  let row = 1
  let col = 1
  let newline = true
  let state = CONTENT
  let start = 0
  let consumingSameLineContent = false
  let escapedInlineCounter = 0
  let escapedParamCounter = 0
  let consumingEscapedParams = false
  let consumingUnparsed = false // true when in an @@ block
  let lastInlineContentEndPos = void (0)
  let unparsedIndentStart = void (0)

  const indent = [0]
  const tokens = []

  function emit (type, value) {
    if (arguments.length > 1) {
      tokens.push({type: type, value: value})
    } else {
      tokens.push({type: type})
    }
  }

  function consume (type, next) {
    emit(type, str.substring(start, pos))
    state = next
    start = pos + 1
  }

  function consumeIfNonEmpty (type, next, escaped) {
    let v = str.substring(start, pos)
    if (v.length > 0) {
      if (escaped) {
        const l = escaped.length
        for (let i = 0; i < l; i++) {
          const es = escaped[i]
          v = v.replace(es, es[1])
        }
      }
      emit(type, v)
    }
    state = next
    start = pos + 1
  }

  function shuffleIfNext (character) {
    if (str[pos + 1] === character) {
      pos++
      start++
      return true
    } else {
      return false
    }
  }

  function err (msg) {
    let start = str.lastIndexOf('\n', pos - 1)
    start = str.lastIndexOf('\n', start - 1)
    start = str.lastIndexOf('\n', start - 1)

    const nextNewline = str.indexOf('\n', pos)
    const errorLineEnd = nextNewline === -1 ? pos : nextNewline

    let end = str.indexOf('\n', errorLineEnd + 1)
    if (end === -1) {
      end = str.length - 1
    } else {
      end = str.indexOf('\n', end + 1)
      if (end === -1) {
        end = str.length - 1
      } else {
        end = str.indexOf('\n', end + 1)
        if (end === -1) {
          end = str.length - 1
        }
      }
    }

    const indentSpaces = (new Array(col)).join(' ')

    const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg
    const context = str.substring(start, errorLineEnd) + '\n' + indentSpaces + '^^^^^^^^\n' + str.substring(errorLineEnd + 1, end)

    throw new ParseError(message + '\n' + context, context, str, row, col, msg, pos)
  }

  while (pos < str.length) {
    // indentation and comment handling.
    if (newline) {
      while (newline && pos < str.length) {
        let ind = 0
        while (str[pos + ind] === ' ' && pos + ind < str.length) {
          ind++
        }

        // if it was an empty content line, then add it to the list of empty
        // content to emit on the next non empty line
        if (str[pos + ind] === '\n') {
          pos += ind + 1
          row++
          emit('EMPTY_CONTENT', str.substring(start, pos - 1))
        } else if (str[pos + ind] !== '#' || consumingUnparsed) {
          if (ind > indent[indent.length - 1]) {
            emit('INDENT', ind - indent[indent.length - 1])
            indent.push(ind)
          } else if (ind < indent[indent.length - 1]) {
            while (indent[indent.length - 1] > ind) {
              const prev = indent.pop()
              emit('DEDENT', prev - indent[indent.length - 1])

              if (consumingUnparsed && unparsedIndentStart >= indent[indent.length - 1]) {
                consumingUnparsed = false
              }
            }

            if (indent.length > 0 && (indent[indent.length - 1] !== ind)) {
              err('indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent[indent.length - 1] + ' spaces was expected')
            }
          }
          pos += ind

          if (str[pos] === '\\' && str[pos + 1] === '#') {
            pos++
          }
          newline = false
        } else {
          while (str[pos] !== '\n' && pos < str.length) {
            pos++
          }
          emit('COMMENT', str.substring(start + ind + 1, pos))
          pos++
          row++
        }

        start = pos
      }
    }

    if (pos >= str.length) {
      return tokens
    }

    const s = str[pos]

    if (state === TYPE) {
      if (s === ' ') {
        consume('TYPE', PARAMS)
        while (shuffleIfNext(' ') && pos < str.length) {}
      } else if (s === ':') {
        consume('TYPE', CONTENT)
        shuffleIfNext(' ')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '[') {
        consume('TYPE', INLINE_CONTENT)
        emit('START_INLINE_CONTENT')
      } else if (s === '(') {
        consume('TYPE', INLINE_PARAMS)
      } else if (s === '\n') {
        consume('TYPE', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === PARAMS) {
      if (s === '[') {
        escapedParamCounter++
        consumingEscapedParams = true
      } else if (consumingEscapedParams && (s === ']')) {
        escapedParamCounter--
        if (escapedParamCounter === 0) {
          consumingEscapedParams = false
        }
      } else if (s === ':' && !consumingEscapedParams) {
        consume('PARAMS', CONTENT)
        shuffleIfNext(' ')
        shuffleIfNext(']')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '\n') {
        escapedParamCounter = 0
        consumingEscapedParams = false
        consume('PARAMS', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === CONTENT) {
      if (s === '@' && !consumingUnparsed) {
        consumeIfNonEmpty('CONTENT', TYPE)
        if (str[pos + 1] === '@') {
          consumingUnparsed = true
          unparsedIndentStart = indent[indent.length - 1]
          pos++
          start++
        }
      } else if (s === '\n') {
        if (consumingSameLineContent) {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
          consumingUnparsed = false
        } else {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
        }
      }
    } else if (state === INLINE_PARAMS) {
      if (s === ')') {
        if (str[pos + 1] === '[') {
          consume('PARAMS', INLINE_CONTENT)
          emit('START_INLINE_CONTENT')
          start++
          pos++
        } else {
          consume('PARAMS', CONTENT)
        }
      }
    } else { // if (state === INLINE_CONTENT) {
      if (s === '\\' && (str[pos + 1] === '[' || str[pos + 1] === ']')) {
        pos++
      } else if (s === '[') {
        escapedInlineCounter++
      } else if (s === ']') {
        if (escapedInlineCounter === 0) {
          consumeIfNonEmpty('CONTENT', CONTENT, ['\\]', '\\['])
          emit('END_INLINE_CONTENT')
          lastInlineContentEndPos = pos + 1
        } else {
          escapedInlineCounter--
        }
      } else if (s === '\n') {
        consume('CONTENT', INLINE_CONTENT)
      }
    }

    if (s === '\n') {
      col = 0
      row++
      newline = true
    } else {
      col++
    }

    pos++
  }

  if (state === TYPE) {
    emit('TYPE', str.substring(start, pos))
  } else if (state === PARAMS) {
    emit('PARAMS', str.substring(start, pos))
  } else if (state === INLINE_PARAMS) {
    err('missing closing ) bracket?')
  } else if (state === INLINE_CONTENT) {
    err('missing closing ] bracket?')
  } else { // if (state === CONTENT) {
    if (start < pos) {
      emit('CONTENT', str.substring(start, pos))
    }
  }

  return tokens
}
1297009
function Lexer(text, cfg) {
        var self = this;
        self.page = new Page(text);
        self.cursor = new Cursor();
        self.nodeFactory = this;
        this.cfg = cfg || {};
    }
121954
function eatUnquoted(stream) {
	const start = stream.pos;
	if (stream.eatWhile(isUnquoted)) {
		stream.start = start;
		return true;
	}
}
579234
Token match( TokenStream input,
                 int ttype,
                 String text,
                 int[] follow,
                 DroolsEditorType etype ) throws RecognitionException {
        Token matchedSymbol = null;
        matchedSymbol = input.LT(1);
        if (input.LA(1) == ttype && (text == null || text.equals(matchedSymbol.getText()))) {
            input.consume();
            state.errorRecovery = false;
            state.failed = false;
            helper.emit(matchedSymbol,
                    etype);
            return matchedSymbol;
        }
        if (state.backtracking > 0) {
            state.failed = true;
            return matchedSymbol;
        }
        matchedSymbol = recoverFromMismatchedToken(input,
                ttype,
                text,
                follow);
        helper.emit(matchedSymbol,
                etype);
        return matchedSymbol;
    }
790099
void waitNewLeaderAckFromQuorum()
      throws TimeoutException, InterruptedException, IOException {
    LOG.debug("Waiting for synchronization to followers complete.");
    int completeCount = 0;
    Zxid lastZxid = persistence.getLatestZxid();
    while (completeCount < this.quorumMap.size()) {
      // Here we should use sync_timeout.
      MessageTuple tuple = filter.getExpectedMessage(MessageType.ACK, null,
                                                     getSyncTimeoutMs());
      ZabMessage.Ack ack = tuple.getMessage().getAck();
      String source =tuple.getServerId();
      Zxid zxid = MessageBuilder.fromProtoZxid(ack.getZxid());
      if (!this.quorumMap.containsKey(source)) {
        LOG.warn("Quorum set doesn't contain {}, a bug?", source);
        continue;
      }
      if (zxid.compareTo(lastZxid) != 0) {
        LOG.error("The follower {} is not correctly synchronized.", source);
        throw new RuntimeException("The synchronized follower's last zxid"
            + "doesn't match last zxid of current leader.");
      }
      PeerHandler ph = this.quorumMap.get(source);
      ph.setLastAckedZxid(zxid);
      completeCount++;
    }
  }
1245316
def _end_escape_sequence(self, char):
        

        num = ord(char)
        if num in self.sequence:
            self.dispatch(self.sequence[num], *self.params)
        self.state = "stream"
        self.current_param = ""
        self.params = []
398055
public void consumeUntil(Parser recognizer, IntervalSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) {
        CSSToken t;
        boolean finish;
        TokenStream input = recognizer.getInputStream();
        do {
            Token next = input.LT(1);
            if (next instanceof CSSToken) {
                t = (CSSToken) input.LT(1);
                if (t.getType() == Token.EOF) {
                    logger.trace("token eof ");
                    break;
                }
            } else
                break; /* not a CSSToken, probably EOF */
            // consume token if does not match
            finish = (t.getLexerState().isBalanced(mode, ls, t) && follow.contains(t.getType()));
            if (!finish) {
                logger.trace("Skipped: {}", t);
                input.consume();
            }
        } while (!finish);
    }
719338
@Override
    public int following(int offset) {
        CharacterIterator text = getText();

        // if we have no cached break positions, or if "offset" is outside the
        // range covered by the cache, then dump the cache and call our
        // inherited following() method.  This will call other methods in this
        // class that may refresh the cache.
        if (fCachedBreakPositions == null || offset < fCachedBreakPositions[0] ||
                offset >= fCachedBreakPositions[fCachedBreakPositions.length - 1]) {
            fCachedBreakPositions = null;
            return rulesFollowing(offset);
        }

        // on the other hand, if "offset" is within the range covered by the
        // cache, then just search the cache for the first break position
        // after "offset"
        else {
            fPositionInCache = 0;
            while (fPositionInCache < fCachedBreakPositions.length
                   && offset >= fCachedBreakPositions[fPositionInCache])
                ++fPositionInCache;
            text.setIndex(fCachedBreakPositions[fPositionInCache]);
            return text.getIndex();
        }
    }
1225015
function readNext() {
    let stream, state;

    while (!isDone) {
      if (!state1.ended
          && (state2.ended || state1.totalDataLen <= state2.totalDataLen)) {
        stream = stream1;
        state = state1;
      } else if (!state2.ended) {
        stream = stream2;
        state = state2;
      } else {
        debug('All streams have ended.  No further reads.');
        return;
      }

      const data = stream.read();
      if (data === null) {
        debug(`Waiting for ${streamName(stream)} to be readable...`);
        stream.once('readable', readNext);
        return;
      }

      handleData.call(stream, state, data);
    }
  }
1217130
function (str, callback) {
            var root, start, end, zone, line, lines, buff = [], c, error = null;

            i = j = current = furthest = 0;
            input = str.replace(/\r\n/g, '\n');

            // Remove potential UTF Byte Order Mark
            input = input.replace(/^\uFEFF/, '');

            // Split the input into chunks.
            chunks = (function (chunks) {
                var j = 0,
                    skip = /(?:@\{[\w-]+\}|[^"'`\{\}\/\(\)\\])+/g,
                    comment = /\/\*(?:[^*]|\*+[^\/*])*\*+\/|\/\/.*/g,
                    string = /"((?:[^"\\\r\n]|\\.)*)"|'((?:[^'\\\r\n]|\\.)*)'|`((?:[^`]|\\.)*)`/g,
                    level = 0,
                    match,
                    chunk = chunks[0],
                    inParam;

                for (var i = 0, c, cc; i < input.length;) {
                    skip.lastIndex = i;
                    if (match = skip.exec(input)) {
                        if (match.index === i) {
                            i += match[0].length;
                            chunk.push(match[0]);
                        }
                    }
                    c = input.charAt(i);
                    comment.lastIndex = string.lastIndex = i;

                    if (match = string.exec(input)) {
                        if (match.index === i) {
                            i += match[0].length;
                            chunk.push(match[0]);
                            continue;
                        }
                    }

                    if (!inParam && c === '/') {
                        cc = input.charAt(i + 1);
                        if (cc === '/' || cc === '*') {
                            if (match = comment.exec(input)) {
                                if (match.index === i) {
                                    i += match[0].length;
                                    chunk.push(match[0]);
                                    continue;
                                }
                            }
                        }
                    }
                    
                    switch (c) {
                        case '{': if (! inParam) { level ++;        chunk.push(c);                           break }
                        case '}': if (! inParam) { level --;        chunk.push(c); chunks[++j] = chunk = []; break }
                        case '(': if (! inParam) { inParam = true;  chunk.push(c);                           break }
                        case ')': if (  inParam) { inParam = false; chunk.push(c);                           break }
                        default:                                    chunk.push(c);
                    }
                    
                    i++;
                }
                if (level != 0) {
                    error = new(LessError)({
                        index: i-1,
                        type: 'Parse',
                        message: (level > 0) ? "missing closing `}`" : "missing opening `{`",
                        filename: env.filename
                    }, env);
                }

                return chunks.map(function (c) { return c.join('') });;
            })([[]]);

            if (error) {
                return callback(error, env);
            }

            // Start with the primary rule.
            // The whole syntax tree is held under a Ruleset node,
            // with the `root` property set to true, so no `{}` are
            // output. The callback is called when the input is parsed.
            try {
                root = new(tree.Ruleset)([], $(this.parsers.primary));
                root.root = true;
            } catch (e) {
                return callback(new(LessError)(e, env));
            }

            root.toCSS = (function (evaluate) {
                var line, lines, column;

                return function (options, variables) {
                    var frames = [], importError;

                    options = options || {};
                    //
                    // Allows setting variables with a hash, so:
                    //
                    //   `{ color: new(tree.Color)('#f01') }` will become:
                    //
                    //   new(tree.Rule)('@color',
                    //     new(tree.Value)([
                    //       new(tree.Expression)([
                    //         new(tree.Color)('#f01')
                    //       ])
                    //     ])
                    //   )
                    //
                    if (typeof(variables) === 'object' && !Array.isArray(variables)) {
                        variables = Object.keys(variables).map(function (k) {
                            var value = variables[k];

                            if (! (value instanceof tree.Value)) {
                                if (! (value instanceof tree.Expression)) {
                                    value = new(tree.Expression)([value]);
                                }
                                value = new(tree.Value)([value]);
                            }
                            return new(tree.Rule)('@' + k, value, false, 0);
                        });
                        frames = [new(tree.Ruleset)(null, variables)];
                    }

                    try {
                        var css = evaluate.call(this, { frames: frames })
                                          .toCSS([], { compress: options.compress || false, dumpLineNumbers: env.dumpLineNumbers });
                    } catch (e) {
                        throw new(LessError)(e, env);
                    }

                    if ((importError = parser.imports.error)) { // Check if there was an error during importing
                        if (importError instanceof LessError) throw importError;
                        else                                  throw new(LessError)(importError, env);
                    }

                    if (options.yuicompress && less.mode === 'node') {
                        return require('ycssmin').cssmin(css);
                    } else if (options.compress) {
                        return css.replace(/(\s)+/g, "$1");
                    } else {
                        return css;
                    }
                };
            })(root.eval);

            // If `i` is smaller than the `input.length - 1`,
            // it means the parser wasn't able to parse the whole
            // string, so we've got a parsing error.
            //
            // We try to extract a \n delimited string,
            // showing the line where the parse error occured.
            // We split it up into two parts (the part which parsed,
            // and the part which didn't), so we can color them differently.
            if (i < input.length - 1) {
                i = furthest;
                lines = input.split('\n');
                line = (input.slice(0, i).match(/\n/g) || "").length + 1;

                for (var n = i, column = -1; n >= 0 && input.charAt(n) !== '\n'; n--) { column++ }

                error = {
                    type: "Parse",
                    message: "Syntax Error on line " + line,
                    index: i,
                    filename: env.filename,
                    line: line,
                    column: column,
                    extract: [
                        lines[line - 2],
                        lines[line - 1],
                        lines[line]
                    ]
                };
            }

            if (this.imports.queue.length > 0) {
                finish = function (e) {
                    e = error || e;
                    if (e) callback(e);
                    else callback(null, root);
                };
            } else {
                callback(error, root);
            }
        }
691427
function lex(source) {
    var tokens = [];
    var chunk = clean(source);
    var total = chunk.length;
    var iterations = 0;

    while (chunk.length > 0) {
        for (var regexpName in REGEXPS) {
            var regexp = REGEXPS[regexpName];
            var match = regexp.exec(chunk);

            if (match) {
                var original = match[0];
                var str = original;
                if (regexpName === 'string') str = str.replace(/\\"/g, '"');
                tokens.push({ type: regexpName, string: str, original: original });
                // Need to slice the chunk at the original match length
                chunk = chunk.slice(match[0].length, chunk.length);
                break;
            }
        }

        // We've probably failed to parse correctly if we get here
        if (iterations++ > total) {
            var parts = [
                'Runiq: Lexer ran too many iterations!',
                '--- Failed at chunk: `' + chunk + '`'
            ];
            throw new Error(parts.join('\n'));
        }
    }

    return tokens;
}
1329124
@Requires("nextToken == null")
  @Ensures("result == (nextToken != null)")
  protected boolean lex() throws IOException {
    int startOffset = currentOffset;
    StringBuilder buffer = new StringBuilder();
    int c = readChar(true);
    if (c == -1) {
      return false;
    }
    buffer.append((char) c);
    switch (c) {
      case '/':
        /* Comments. */
        c = readChar(true);
        switch (c) {
          case '/':
            buffer.append((char) c);
            do {
              c = readChar(false);
              buffer.append((char) c);
            } while (c != '\n');
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          case '*':
            buffer.append((char) c);
            for (;;) {
              c = readChar(false);
              buffer.append((char) c);
              if (c == '*') {
                c = readChar(false);
                buffer.append((char) c);
                if (c == '/') {
                  break;
                }
              }
            }
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          default:
            unreadChar(c);
            nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                  startOffset);
        }
        break;

      case '\'':
      case '"':
        /* Quoted strings. */
        int delim = c;
        for (;;) {
          c = readChar(false);
          buffer.append((char) c);
          if (c == delim) {
            break;
          }
          if (c == '\\') {
            buffer.append((char) readChar(false));
          }
        }
        nextToken = new Token(TokenKind.QUOTE, buffer.toString(), startOffset);
        break;

      default:
        if (Character.isJavaIdentifierStart(c)) {
          /* Identifiers. */
          while ((c = readChar(true)) != -1
                 && Character.isJavaIdentifierPart(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.WORD, buffer.toString(), startOffset);
        } else if (Character.isWhitespace(c)) {
          /* White space. */
          while ((c = readChar(true)) != -1 && Character.isWhitespace(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.SPACE, buffer.toString(),
                                startOffset);
        } else {
          /* Symbol. */
          nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                startOffset);
        }
    }
    return true;
  }
554416
def follow(self, delay=1.0):
        
        trailing = True

        while 1:
            where = self.file.tell()
            line = self.file.readline()
            if line:
                if trailing and line in self.line_terminators:
                    # This is just the line terminator added to the end of the file
                    # before a new line, ignore.
                    trailing = False
                    continue

                if line[-1] in self.line_terminators:
                    line = line[:-1]
                    if line[-1:] == '\r\n' and '\r\n' in self.line_terminators:
                        # found crlf
                        line = line[:-1]

                trailing = False
                yield line
            else:
                trailing = True
                self.seek(where)
                time.sleep(delay)
633747
@Override
    protected void handleTransliterate(Replaceable text,
                                       Position index, boolean incremental) {
        // Our caller (filteredTransliterate) has already narrowed us
        // to an unfiltered run.  Delete it.
        text.replace(index.start, index.limit, "");
        int len = index.limit - index.start;
        index.contextLimit -= len;
        index.limit -= len;
    }
153123
private void flushCharacters()
    {
        int indentLength = 0;
        int len = charBuffer.length();
        /*
         * Count characters from end of ignorable whitespace to first end of line we hit
         */
        for ( int i = len - 1; i >= 0; i-- )
        {
            char ch = charBuffer.charAt( i );
            switch ( ch )
            {
                case '\n':
                case '\r':
                    lastIndent = new Indent( charLineNumber, indentLength );
                    charBuffer.setLength( 0 );
                    return;
                case ' ':
                case '\t':
                    indentLength++;
                    break;
                default:
                    /*
                     * No end of line foundIndent in the trailing whitespace. Leave the foundIndent from previous
                     * ignorable whitespace unchanged
                     */
                    charBuffer.setLength( 0 );
                    return;
            }
        }
    }