{ function_description }
@memberof   module:wick~internals.compiler
@param      {Lexer}  lex     The lex
@return     {Array}   an
504762
function evaluate(lex, EVENT$$1 = false) {

    let binds = [];


    lex.IWS = false;

    let start = lex.pos;
    while (!lex.END && lex.ty !== lex.types.str) {
        switch (lex.ch) {
            case barrier_a_start:
                if (lex.pk.ch == barrier_b_start || lex.p.ch == barrier_a_start) {

                    let sentinel = (lex.p.ch == barrier_a_start) ? barrier_a_end : barrier_b_end;

                    let pk2 = lex.p.pk;


                    if (pk2.ch == barrier_b_start) {
                        sentinel = barrier_b_end;
                        if (start < lex.p.pos)
                            binds.push(new RawValueBinding(lex.p.slice(start)));

                        lex.p.sync();
                    } else if (start < lex.pos) {

                        binds.push(new RawValueBinding(lex.slice(start)));
                    } //create text node


                    lex.sync().n;
                    lex.IWS = true; // Do not produce white space tokens during this portion.
                    let pk = lex.pk;
                    let Message= false;


                    while (!pk.END && (pk.ch !== sentinel || (pk.pk.ch !== barrier_a_end && pk.p.ch !== barrier_a_start) || (pk.p.n.ch === barrier_a_end))) { 
                        let prev = pk.ch;
                        pk.n; 
                        if(pk.ch == barrier_a_start && prev == barrier_a_end)
                            Message =true;
                    }


                    if (lex.tl < pk.off - lex.off - 1 && !Message) {
                        /***** Start Expression *******/

                        const elex = lex.copy(); //The expression Lexer

                        elex.fence(pk);

                        lex.sync();

                        if (pk.END) //Should still have `))` or `|)` in the input string
                            throw new Error("Should be more to this!");

                        processExpression(elex, binds);

                        lex.a(sentinel);
                        /***** End Expression ********/
                    } else {

                        /************************** Start Single Identifier Binding *******************************/
                        let id = lex.tx;
                        let binding = new DynamicBinding();
                        binding.tap_name = id;
                        let index = binds.push(binding) - 1;
                        lex.n.a(sentinel);

                        if (EVENT$$1) {
                            /***************************** Looking for Event Bindings ******************************************/
                            
                            if (lex.ch == barrier_a_start || lex.ch == barrier_b_start) {

                                binds[index] = new EventBinding(binds[index]);

                                let sentinel = (lex.ch == barrier_a_start) ? barrier_a_end : barrier_b_end;

                                lex.IWS = true; // Do not produce white space tokens during this portion.

                                let pk = lex.pk;

                                while (!pk.END && (pk.ch !== sentinel || (pk.pk.ch !== barrier_a_end))) { pk.n; }

                                lex.n;

                                if (lex.tl < pk.off - lex.off || BannedIdentifiers[lex.tx]) {

                                    const elex = lex.copy(); //The expression Lexer

                                    elex.fence(pk);

                                    lex.sync();

                                    if (pk.END) //Should still have `))` or `|)` in the input string
                                        throw new Error("Should be more to this!");

                                    const event_binds = [];

                                    processExpression(elex, event_binds);

                                    binds[index].bind = event_binds[0];

                                    lex.a(sentinel);

                                } else {
                                    if (lex.ch !== sentinel) {
                                        let id = lex.tx,
                                            binding;
                                        if (lex.ty !== lex.types.id) {
                                            switch (lex.ty) {
                                                case lex.types.num:
                                                    binding = new RawValueBinding(parseFloat(id));
                                                    break;
                                                case lex.types.str:
                                                    binding = new RawValueBinding(id.slice(1, -1));
                                                    break;
                                                default:
                                                    binding = new RawValueBinding(id.slice);
                                            }
                                        } else {
                                            binding = new DynamicBinding();
                                            binding.tap_name = id;
                                        }
                                        binds[index].bind = binding;
                                        lex.n;
                                    }
                                    lex.a(sentinel);
                                }
                            }
                        }
                    }

                    lex.IWS = false;
                    
                    start = lex.off + 1; //Should at the sentinel.
                    
                    lex.a(barrier_a_end);
                    
                    continue;
                }
                break;
        }
        lex.n;
    }

    if (start < lex.off) {
        lex.off = start;
        lex.END = false;
        lex.tl = 0;
        lex.END = false;

        let DATA_END = start;

        while (!lex.n.END)
            if (!(lex.ty & (lex.types.ws | lex.types.nl)))
                DATA_END = lex.off + lex.tl;
            
        if (DATA_END > start) {
            lex.sl = DATA_END;
            binds.push(new RawValueBinding(lex.slice(start)));
        }
    }

    return binds;
}
230178
function(code) {
    var modules = [];

    var lex = Espruino.Core.Utils.getLexer(code);
    var tok = lex.next();
    var state = 0;
    while (tok!==undefined) {
      if (state==0 && tok.str=="require") {
        state=1;
      } else if (state==1 && tok.str=="(") {
        state=2;
      } else if (state==2 && (tok.type=="STRING")) {
        state=0;
        var module = tok.value;
        if (!isBuiltIn(module) && modules.indexOf(module)<0)
          modules.push(module);
      } else
        state = 0;
      tok = lex.next();
    }

    return modules;
  }
649045
function genLex (lex) {
    var s = [];

    if (lex.macros) {
        for (var macro in lex.macros) if (lex.macros.hasOwnProperty(macro)) {
            s.push(macro, '         ', lex.macros[macro], '\n');
        }
    }
    if (lex.actionInclude) {
        s.push('\n%{\n', lex.actionInclude, '\n%}\n');
    }
    s.push('\n%%\n');
    if (lex.rules) {
        for (var rule;rule=lex.rules.shift();) {
            s.push(genLexRegex(rule[0]), '         ', genLexRule(rule[1]), '\n');
        }
    }
    s.push('\n');

    return s.join('');
}
691427
function lex(source) {
    var tokens = [];
    var chunk = clean(source);
    var total = chunk.length;
    var iterations = 0;

    while (chunk.length > 0) {
        for (var regexpName in REGEXPS) {
            var regexp = REGEXPS[regexpName];
            var match = regexp.exec(chunk);

            if (match) {
                var original = match[0];
                var str = original;
                if (regexpName === 'string') str = str.replace(/\\"/g, '"');
                tokens.push({ type: regexpName, string: str, original: original });
                // Need to slice the chunk at the original match length
                chunk = chunk.slice(match[0].length, chunk.length);
                break;
            }
        }

        // We've probably failed to parse correctly if we get here
        if (iterations++ > total) {
            var parts = [
                'Runiq: Lexer ran too many iterations!',
                '--- Failed at chunk: `' + chunk + '`'
            ];
            throw new Error(parts.join('\n'));
        }
    }

    return tokens;
}
860200
def lex(filename):
    """"""
    with io.open(filename, mode='r', encoding='utf-8') as f:
        it = _lex_file_object(f)
        it = _balance_braces(it, filename)
        for token, line, quoted in it:
            yield (token, line, quoted)
758604
def str_lexer(mode):
    
    cast_to_const = ConstStrPool.cast_to_const

    def f_raw(inp_str, pos):
        return cast_to_const(mode) if inp_str.startswith(mode, pos) else None

    def f_collection(inp_str, pos):
        for each in mode:
            if inp_str.startswith(each, pos):
                return cast_to_const(each)
        return None

    if isinstance(mode, str):
        return f_raw

    if len(mode) is 1:
        mode = mode[0]
        return f_raw

    return f_collection
1106328
function Lexer(str, options) {
  options = options || {};
  if (typeof str !== 'string') {
    throw new Error('Expected source code to be a string but got "' + (typeof str) + '"')
  }
  if (typeof options !== 'object') {
    throw new Error('Expected "options" to be an object but got "' + (typeof options) + '"')
  }
  //Strip any UTF-8 BOM off of the start of `str`, if it exists.
  str = str.replace(/^\uFEFF/, '');
  this.input = str.replace(/\r\n|\r/g, '\n');
  this.originalInput = this.input;
  this.filename = options.filename;
  this.interpolated = options.interpolated || false;
  this.lineno = options.startingLine || 1;
  this.colno = options.startingColumn || 1;
  this.plugins = options.plugins || [];
  this.indentStack = [0];
  this.indentRe = null;
  // If #{} or !{} syntax is allowed when adding text
  this.interpolationAllowed = true;

  this.tokens = [];
  this.ended = false;
}
572721
function parseUnaryExpression() {
        var token, expr;

        if (match('++') || match('--')) {
            token = lex();
            expr = parseUnaryExpression();
            // 11.4.4, 11.4.5
            if (strict && expr.type === Syntax.Identifier && isRestrictedWord(expr.name)) {
                throwError({}, Messages.StrictLHSPrefix);
            }
            expr = {
                type: Syntax.UpdateExpression,
                operator: token.value,
                argument: expr,
                prefix: true
            };
            return expr;
        }

        if (match('+') || match('-') || match('~') || match('!')) {
            expr = {
                type: Syntax.UnaryExpression,
                operator: lex().value,
                argument: parseUnaryExpression()
            };
            return expr;
        }

        if (matchKeyword('delete') || matchKeyword('void') || matchKeyword('typeof')) {
            expr = {
                type: Syntax.UnaryExpression,
                operator: lex().value,
                argument: parseUnaryExpression()
            };
            if (strict && expr.operator === 'delete' && expr.argument.type === Syntax.Identifier) {
                throwError({}, Messages.StrictDelete);
            }
            return expr;
        }

        return parsePostfixExpression();
    }
603419
function Lexer (options) {
  this.tokens = []
  this.options = options || defaults
  this.rules = block
  if (this.options.gfm) {
    Object.assign(this.rules, block.opts.gfm)
  }
  if (this.options.include) {
    Object.assign(this.rules, block.opts.include)
  }
  if (this.options.numberedheadings) {
    Object.assign(this.rules, block.opts.numberedheadings)
  }
  if (this.options.toc) {
    Object.assign(this.rules, block.opts.toc)
  }
  if (this.options.ref) {
    Object.assign(this.rules, block.opts.ref)
  }
}
821248
function parser(text, json, $filter, csp){
  var ZERO = valueFn(0),
      value,
      tokens = lex(text, csp),
      assignment = _assignment,
      functionCall = _functionCall,
      fieldAccess = _fieldAccess,
      objectIndex = _objectIndex,
      filterChain = _filterChain;

  if(json){
    // The extra level of aliasing is here, just in case the lexer misses something, so that
    // we prevent any accidental execution in JSON.
    assignment = logicalOR;
    functionCall =
      fieldAccess =
      objectIndex =
      filterChain =
        function() { throwError("is not valid json", {text:text, index:0}); };
    value = primary();
  } else {
    value = statements();
  }
  if (tokens.length !== 0) {
    throwError("is an unexpected token", tokens[0]);
  }
  return value;

  ///////////////////////////////////
  function throwError(msg, token) {
    throw Error("Syntax Error: Token '" + token.text +
      "' " + msg + " at column " +
      (token.index + 1) + " of the expression [" +
      text + "] starting at [" + text.substring(token.index) + "].");
  }

  function peekToken() {
    if (tokens.length === 0)
      throw Error("Unexpected end of expression: " + text);
    return tokens[0];
  }

  function peek(e1, e2, e3, e4) {
    if (tokens.length > 0) {
      var token = tokens[0];
      var t = token.text;
      if (t==e1 || t==e2 || t==e3 || t==e4 ||
          (!e1 && !e2 && !e3 && !e4)) {
        return token;
      }
    }
    return false;
  }

  function expect(e1, e2, e3, e4){
    var token = peek(e1, e2, e3, e4);
    if (token) {
      if (json && !token.json) {
        throwError("is not valid json", token);
      }
      tokens.shift();
      return token;
    }
    return false;
  }

  function consume(e1){
    if (!expect(e1)) {
      throwError("is unexpected, expecting [" + e1 + "]", peek());
    }
  }

  function unaryFn(fn, right) {
    return function(self, locals) {
      return fn(self, locals, right);
    };
  }

  function binaryFn(left, fn, right) {
    return function(self, locals) {
      return fn(self, locals, left, right);
    };
  }

  function statements() {
    var statements = [];
    while(true) {
      if (tokens.length > 0 && !peek('}', ')', ';', ']'))
        statements.push(filterChain());
      if (!expect(';')) {
        // optimize for the common case where there is only one statement.
        // TODO(size): maybe we should not support multiple statements?
        return statements.length == 1
          ? statements[0]
          : function(self, locals){
            var value;
            for ( var i = 0; i < statements.length; i++) {
              var statement = statements[i];
              if (statement)
                value = statement(self, locals);
            }
            return value;
          };
      }
    }
  }

  function _filterChain() {
    var left = expression();
    var token;
    while(true) {
      if ((token = expect('|'))) {
        left = binaryFn(left, token.fn, filter());
      } else {
        return left;
      }
    }
  }

  function filter() {
    var token = expect();
    var fn = $filter(token.text);
    var argsFn = [];
    while(true) {
      if ((token = expect(':'))) {
        argsFn.push(expression());
      } else {
        var fnInvoke = function(self, locals, input){
          var args = [input];
          for ( var i = 0; i < argsFn.length; i++) {
            args.push(argsFn[i](self, locals));
          }
          return fn.apply(self, args);
        };
        return function() {
          return fnInvoke;
        };
      }
    }
  }

  function expression() {
    return assignment();
  }

  function _assignment() {
    var left = logicalOR();
    var right;
    var token;
    if ((token = expect('='))) {
      if (!left.assign) {
        throwError("implies assignment but [" +
          text.substring(0, token.index) + "] can not be assigned to", token);
      }
      right = logicalOR();
      return function(self, locals){
        return left.assign(self, right(self, locals), locals);
      };
    } else {
      return left;
    }
  }

  function logicalOR() {
    var left = logicalAND();
    var token;
    while(true) {
      if ((token = expect('||'))) {
        left = binaryFn(left, token.fn, logicalAND());
      } else {
        return left;
      }
    }
  }

  function logicalAND() {
    var left = equality();
    var token;
    if ((token = expect('&&'))) {
      left = binaryFn(left, token.fn, logicalAND());
    }
    return left;
  }

  function equality() {
    var left = relational();
    var token;
    if ((token = expect('==','!='))) {
      left = binaryFn(left, token.fn, equality());
    }
    return left;
  }

  function relational() {
    var left = additive();
    var token;
    if ((token = expect('<', '>', '<=', '>='))) {
      left = binaryFn(left, token.fn, relational());
    }
    return left;
  }

  function additive() {
    var left = multiplicative();
    var token;
    while ((token = expect('+','-'))) {
      left = binaryFn(left, token.fn, multiplicative());
    }
    return left;
  }

  function multiplicative() {
    var left = unary();
    var token;
    while ((token = expect('*','/','%'))) {
      left = binaryFn(left, token.fn, unary());
    }
    return left;
  }

  function unary() {
    var token;
    if (expect('+')) {
      return primary();
    } else if ((token = expect('-'))) {
      return binaryFn(ZERO, token.fn, unary());
    } else if ((token = expect('!'))) {
      return unaryFn(token.fn, unary());
    } else {
      return primary();
    }
  }


  function primary() {
    var primary;
    if (expect('(')) {
      primary = filterChain();
      consume(')');
    } else if (expect('[')) {
      primary = arrayDeclaration();
    } else if (expect('{')) {
      primary = object();
    } else {
      var token = expect();
      primary = token.fn;
      if (!primary) {
        throwError("not a primary expression", token);
      }
    }

    var next, context;
    while ((next = expect('(', '[', '.'))) {
      if (next.text === '(') {
        primary = functionCall(primary, context);
        context = null;
      } else if (next.text === '[') {
        context = primary;
        primary = objectIndex(primary);
      } else if (next.text === '.') {
        context = primary;
        primary = fieldAccess(primary);
      } else {
        throwError("IMPOSSIBLE");
      }
    }
    return primary;
  }

  function _fieldAccess(object) {
    var field = expect().text;
    var getter = getterFn(field, csp);
    return extend(
        function(self, locals) {
          return getter(object(self, locals), locals);
        },
        {
          assign:function(self, value, locals) {
            return setter(object(self, locals), field, value);
          }
        }
    );
  }

  function _objectIndex(obj) {
    var indexFn = expression();
    consume(']');
    return extend(
      function(self, locals){
        var o = obj(self, locals),
            i = indexFn(self, locals),
            v, p;

        if (!o) return undefined;
        v = o[i];
        if (v && v.then) {
          p = v;
          if (!('$$v' in v)) {
            p.$$v = undefined;
            p.then(function(val) { p.$$v = val; });
          }
          v = v.$$v;
        }
        return v;
      }, {
        assign:function(self, value, locals){
          return obj(self, locals)[indexFn(self, locals)] = value;
        }
      });
  }

  function _functionCall(fn, contextGetter) {
    var argsFn = [];
    if (peekToken().text != ')') {
      do {
        argsFn.push(expression());
      } while (expect(','));
    }
    consume(')');
    return function(self, locals){
      var args = [],
          context = contextGetter ? contextGetter(self, locals) : self;

      for ( var i = 0; i < argsFn.length; i++) {
        args.push(argsFn[i](self, locals));
      }
      var fnPtr = fn(self, locals) || noop;
      // IE stupidity!
      return fnPtr.apply
          ? fnPtr.apply(context, args)
          : fnPtr(args[0], args[1], args[2], args[3], args[4]);
    };
  }

  // This is used with json array declaration
  function arrayDeclaration () {
    var elementFns = [];
    if (peekToken().text != ']') {
      do {
        elementFns.push(expression());
      } while (expect(','));
    }
    consume(']');
    return function(self, locals){
      var array = [];
      for ( var i = 0; i < elementFns.length; i++) {
        array.push(elementFns[i](self, locals));
      }
      return array;
    };
  }

  function object () {
    var keyValues = [];
    if (peekToken().text != '}') {
      do {
        var token = expect(),
        key = token.string || token.text;
        consume(":");
        var value = expression();
        keyValues.push({key:key, value:value});
      } while (expect(','));
    }
    consume('}');
    return function(self, locals){
      var object = {};
      for ( var i = 0; i < keyValues.length; i++) {
        var keyValue = keyValues[i];
        var value = keyValue.value(self, locals);
        object[keyValue.key] = value;
      }
      return object;
    };
  }
}
351657
func lex(input string) []*token {
	var tokens []*token
	l := &lexer{0, 0, 0, len(input), input}
	for {
		token := l.scan()
		if token.mechanism == tEOF {
			break
		}
		tokens = append(tokens, token)
	}
	return tokens
}
766533
function getEncodedLexicalClassifications(text, lexState, syntacticClassifierAbsent) {
            var offset = 0;
            var token = 0 /* Unknown */;
            var lastNonTriviaToken = 0 /* Unknown */;
            // Empty out the template stack for reuse.
            while (templateStack.length > 0) {
                templateStack.pop();
            }
            // If we're in a string literal, then prepend: "\
            // (and a newline).  That way when we lex we'll think we're still in a string literal.
            //
            // If we're in a multiline comment, then prepend: /*
            // (and a newline).  That way when we lex we'll think we're still in a multiline comment.
            switch (lexState) {
                case 3 /* InDoubleQuoteStringLiteral */:
                    text = "\"\\\n" + text;
                    offset = 3;
                    break;
                case 2 /* InSingleQuoteStringLiteral */:
                    text = "'\\\n" + text;
                    offset = 3;
                    break;
                case 1 /* InMultiLineCommentTrivia */:
                    text = "/*\n" + text;
                    offset = 3;
                    break;
                case 4 /* InTemplateHeadOrNoSubstitutionTemplate */:
                    text = "`\n" + text;
                    offset = 2;
                    break;
                case 5 /* InTemplateMiddleOrTail */:
                    text = "}\n" + text;
                    offset = 2;
                // fallthrough
                case 6 /* InTemplateSubstitutionPosition */:
                    templateStack.push(12 /* TemplateHead */);
                    break;
            }
            scanner.setText(text);
            var result = {
                endOfLineState: 0 /* None */,
                spans: []
            };
            // We can run into an unfortunate interaction between the lexical and syntactic classifier
            // when the user is typing something generic.  Consider the case where the user types:
            //
            //      Foo<number
            //
            // From the lexical classifier's perspective, 'number' is a keyword, and so the word will
            // be classified as such.  However, from the syntactic classifier's tree-based perspective
            // this is simply an expression with the identifier 'number' on the RHS of the less than
            // token.  So the classification will go back to being an identifier.  The moment the user
            // types again, number will become a keyword, then an identifier, etc. etc.
            //
            // To try to avoid this problem, we avoid classifying contextual keywords as keywords
            // when the user is potentially typing something generic.  We just can't do a good enough
            // job at the lexical level, and so well leave it up to the syntactic classifier to make
            // the determination.
            //
            // In order to determine if the user is potentially typing something generic, we use a
            // weak heuristic where we track < and > tokens.  It's a weak heuristic, but should
            // work well enough in practice.
            var angleBracketStack = 0;
            do {
                token = scanner.scan();
                if (!ts.isTrivia(token)) {
                    if ((token === 39 /* SlashToken */ || token === 61 /* SlashEqualsToken */) && !noRegexTable[lastNonTriviaToken]) {
                        if (scanner.reScanSlashToken() === 10 /* RegularExpressionLiteral */) {
                            token = 10 /* RegularExpressionLiteral */;
                        }
                    }
                    else if (lastNonTriviaToken === 21 /* DotToken */ && isKeyword(token)) {
                        token = 69 /* Identifier */;
                    }
                    else if (isKeyword(lastNonTriviaToken) && isKeyword(token) && !canFollow(lastNonTriviaToken, token)) {
                        // We have two keywords in a row.  Only treat the second as a keyword if
                        // it's a sequence that could legally occur in the language.  Otherwise
                        // treat it as an identifier.  This way, if someone writes "private var"
                        // we recognize that 'var' is actually an identifier here.
                        token = 69 /* Identifier */;
                    }
                    else if (lastNonTriviaToken === 69 /* Identifier */ &&
                        token === 25 /* LessThanToken */) {
                        // Could be the start of something generic.  Keep track of that by bumping
                        // up the current count of generic contexts we may be in.
                        angleBracketStack++;
                    }
                    else if (token === 27 /* GreaterThanToken */ && angleBracketStack > 0) {
                        // If we think we're currently in something generic, then mark that that
                        // generic entity is complete.
                        angleBracketStack--;
                    }
                    else if (token === 117 /* AnyKeyword */ ||
                        token === 132 /* StringKeyword */ ||
                        token === 130 /* NumberKeyword */ ||
                        token === 120 /* BooleanKeyword */ ||
                        token === 133 /* SymbolKeyword */) {
                        if (angleBracketStack > 0 && !syntacticClassifierAbsent) {
                            // If it looks like we're could be in something generic, don't classify this
                            // as a keyword.  We may just get overwritten by the syntactic classifier,
                            // causing a noisy experience for the user.
                            token = 69 /* Identifier */;
                        }
                    }
                    else if (token === 12 /* TemplateHead */) {
                        templateStack.push(token);
                    }
                    else if (token === 15 /* OpenBraceToken */) {
                        // If we don't have anything on the template stack,
                        // then we aren't trying to keep track of a previously scanned template head.
                        if (templateStack.length > 0) {
                            templateStack.push(token);
                        }
                    }
                    else if (token === 16 /* CloseBraceToken */) {
                        // If we don't have anything on the template stack,
                        // then we aren't trying to keep track of a previously scanned template head.
                        if (templateStack.length > 0) {
                            var lastTemplateStackToken = ts.lastOrUndefined(templateStack);
                            if (lastTemplateStackToken === 12 /* TemplateHead */) {
                                token = scanner.reScanTemplateToken();
                                // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us.
                                if (token === 14 /* TemplateTail */) {
                                    templateStack.pop();
                                }
                                else {
                                    ts.Debug.assert(token === 13 /* TemplateMiddle */, "Should have been a template middle. Was " + token);
                                }
                            }
                            else {
                                ts.Debug.assert(lastTemplateStackToken === 15 /* OpenBraceToken */, "Should have been an open brace. Was: " + token);
                                templateStack.pop();
                            }
                        }
                    }
                    lastNonTriviaToken = token;
                }
                processToken();
            } while (token !== 1 /* EndOfFileToken */);
            return result;
            function processToken() {
                var start = scanner.getTokenPos();
                var end = scanner.getTextPos();
                addResult(start, end, classFromKind(token));
                if (end >= text.length) {
                    if (token === 9 /* StringLiteral */) {
                        // Check to see if we finished up on a multiline string literal.
                        var tokenText = scanner.getTokenText();
                        if (scanner.isUnterminated()) {
                            var lastCharIndex = tokenText.length - 1;
                            var numBackslashes = 0;
                            while (tokenText.charCodeAt(lastCharIndex - numBackslashes) === 92 /* backslash */) {
                                numBackslashes++;
                            }
                            // If we have an odd number of backslashes, then the multiline string is unclosed
                            if (numBackslashes & 1) {
                                var quoteChar = tokenText.charCodeAt(0);
                                result.endOfLineState = quoteChar === 34 /* doubleQuote */
                                    ? 3 /* InDoubleQuoteStringLiteral */
                                    : 2 /* InSingleQuoteStringLiteral */;
                            }
                        }
                    }
                    else if (token === 3 /* MultiLineCommentTrivia */) {
                        // Check to see if the multiline comment was unclosed.
                        if (scanner.isUnterminated()) {
                            result.endOfLineState = 1 /* InMultiLineCommentTrivia */;
                        }
                    }
                    else if (ts.isTemplateLiteralKind(token)) {
                        if (scanner.isUnterminated()) {
                            if (token === 14 /* TemplateTail */) {
                                result.endOfLineState = 5 /* InTemplateMiddleOrTail */;
                            }
                            else if (token === 11 /* NoSubstitutionTemplateLiteral */) {
                                result.endOfLineState = 4 /* InTemplateHeadOrNoSubstitutionTemplate */;
                            }
                            else {
                                ts.Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token);
                            }
                        }
                    }
                    else if (templateStack.length > 0 && ts.lastOrUndefined(templateStack) === 12 /* TemplateHead */) {
                        result.endOfLineState = 6 /* InTemplateSubstitutionPosition */;
                    }
                }
            }
            function addResult(start, end, classification) {
                if (classification === 8 /* whiteSpace */) {
                    // Don't bother with whitespace classifications.  They're not needed.
                    return;
                }
                if (start === 0 && offset > 0) {
                    // We're classifying the first token, and this was a case where we prepended
                    // text.  We should consider the start of this token to be at the start of
                    // the original text.
                    start += offset;
                }
                // All our tokens are in relation to the augmented text.  Move them back to be
                // relative to the original text.
                start -= offset;
                end -= offset;
                var length = end - start;
                if (length > 0) {
                    result.spans.push(start);
                    result.spans.push(length);
                    result.spans.push(classification);
                }
            }
        }
717415
function parseExpression() {
        var expr, startToken = lookahead, expressions;

        expr = isolateCoverGrammar(parseAssignmentExpression);

        if (match(',')) {
            expressions = [expr];

            while (startIndex < length) {
                if (!match(',')) {
                    break;
                }
                lex();
                expressions.push(isolateCoverGrammar(parseAssignmentExpression));
            }

            expr = new WrappingNode(startToken).finishSequenceExpression(expressions);
        }

        return expr;
    }
1170033
def _lex(stream):
    
    lines = enumerate(stream, 1)
    line_no = pos = 0
    try:
        while True:
            if pos == 0:
                line_no, line = next(lines)
            matches = _tdl_lex_re.finditer(line, pos)
            pos = 0  # reset; only used for multiline patterns
            for m in matches:
                gid = m.lastindex
                if gid <= 2:  # potentially multiline patterns
                    if gid == 1:  # docstring
                        s, start_line_no, line_no, line, pos = _bounded(
                            '"""', '"""', line, m.end(), line_no, lines)
                    elif gid == 2:  # comment
                        s, start_line_no, line_no, line, pos = _bounded(
                            '#|', '|#', line, m.end(), line_no, lines)
                    yield (gid, s, line_no)
                    break
                elif gid == 30:
                    raise TdlParsingError(
                        ('Syntax error:\n  {}\n {}^'
                         .format(line, ' ' * m.start())),
                        line_number=line_no)
                else:
                    # token = None
                    # if not (6 < gid < 20):
                    #     token = m.group(gid)
                    token = m.group(gid)
                    yield (gid, token, line_no)
    except StopIteration:
        pass
634046
private String[] getSuffixes(final String lex) {
    final Integer start = Integer.parseInt(this.attributes.get("sufBegin"));
    final Integer end = Integer.parseInt(this.attributes.get("sufEnd"));
    final String[] suffs = new String[end];
    for (int li = start, ll = end; li < ll; li++) {
      suffs[li] = lex.substring(Math.max(lex.length() - li - 1, 0));
    }
    return suffs;
  }
1297009
function Lexer(text, cfg) {
        var self = this;
        self.page = new Page(text);
        self.cursor = new Cursor();
        self.nodeFactory = this;
        this.cfg = cfg || {};
    }
1091484
private String[] getPrefixes(final String lex) {
    final Integer start = Integer.parseInt(this.attributes.get("prefBegin"));
    final Integer end = Integer.parseInt(this.attributes.get("prefEnd"));
    final String[] prefs = new String[end];
    for (int li = start, ll = end; li < ll; li++) {
      prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
    }
    return prefs;
  }