parse pointer into tokens
774681
def parse(self, pointer):
        """"""
        if isinstance(pointer, Pointer):
            return pointer.tokens[:]
        elif pointer == '':
            return []

        tokens = []
        staged, _, children = pointer.partition('/')
        if staged:
            try:
                token = StagesToken(staged)
                token.last = False
                tokens.append(token)
            except ValueError:
                raise ParseError('pointer must start with / or int', pointer)

        if _:
            for part in children.split('/'):
                part = part.replace('~1', '/')
                part = part.replace('~0', '~')
                token = ChildToken(part)
                token.last = False
                tokens.append(token)

        return tokens
324856
def get_tokens(tu, extent):
        
        tokens_memory = POINTER(Token)()
        tokens_count = c_uint()

        conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
                byref(tokens_count))

        count = int(tokens_count.value)

        # If we get no tokens, no memory was allocated. Be sure not to return
        # anything and potentially call a destructor on nothing.
        if count < 1:
            return

        tokens_array = cast(tokens_memory, POINTER(Token * count)).contents

        token_group = TokenGroup(tu, tokens_memory, tokens_count)

        for i in xrange(0, count):
            token = Token()
            token.int_data = tokens_array[i].int_data
            token.ptr_data = tokens_array[i].ptr_data
            token._tu = tu
            token._group = token_group

            yield token
930570
function _parseLeafToken(token) {
  const [, name, remainder] = token.match(LEAF_REGEX);

  return {
    name: name.toLowerCase(),
    params: _parseParams(remainder),
  };
}
1004369
function parse(c) {

  let ret = [];

  // replace windows line breaks with linux line breaks
  let c1 = c.replace(new RegExp('[\r][\n]', 'g'), '\n');

  // split string to line array
  let c2 = c1.split('\n');

  for (let i = 0; i < c2.length; i++) {

    //line object
    let lo = {};

    // line
    let l = c2[i];

    // trim line
    let l1 = l.trim();

    // set line number
    lo.num = i + 1;

    // set original line
    lo.content = l1;

    // check empty line
    if (l1 === '') {
      lo.isEmpty = true;
    } else {
      lo.isEmpty = false;

      // check line comment
      if (l1.startsWith('#')) {
        lo.isComment = true;
        lo.comment = lo.content.substring(1,lo.content.length).trim();
      } else {
        lo.isComment = false;

        // Check included comments
        lo = Object.assign(lo, checkComment(l1));

        // check key value
        lo = Object.assign(lo, checkKeyValue(lo.contentWithoutComment));

        // check block key
        lo = Object.assign(lo, checkBlockKey(lo));
        
    
      }
    }

    //console.log(lo);

    ret.push(lo);
  }

  checkBlockEndSum(ret);
  determineDepth(ret);

  return ret;
}
1157963
function parse (input) {
  var parts = input.split(';');
  var _ = parts.pop();
  var srid = (parts.shift() || '').split('=').pop();

  var i = 0;

  function $ (re) {
    var match = _.substring(i).match(re);
    if (!match) return null;
    else {
      i += match[0].length;
      return match[0];
    }
  }

  function crs (obj) {
    if (obj && srid.match(/\d+/)) {
      obj.crs = {
        type: 'name',
        properties: {
          name: 'urn:ogc:def:crs:EPSG::' + srid
        }
      };
    }

    return obj;
  }

  function white () { $(/^\s*/); }

  function multicoords () {
    white();
    var depth = 0;
    var rings = [];
    var stack = [rings];
    var pointer = rings;
    var elem;

    while (elem =
           $(/^(\()/) ||
             $(/^(\))/) ||
               $(/^(,)/) ||
                 $(tuples)) {
      if (elem === '(') {
        stack.push(pointer);
        pointer = [];
        stack[stack.length - 1].push(pointer);
        depth++;
      } else if (elem === ')') {
        // For the case: Polygon(), ...
        if (pointer.length === 0) return null;

        pointer = stack.pop();
        // the stack was empty, input was malformed
        if (!pointer) return null;
        depth--;
        if (depth === 0) break;
      } else if (elem === ',') {
        pointer = [];
        stack[stack.length - 1].push(pointer);
      } else if (!elem.split(/\s/g).some(isNaN)) {
        Array.prototype.push.apply(pointer, elem.split(/\s/g).map(parseFloat));
      } else {
        return null;
      }
      white();
    }

    if (depth !== 0) return null;

    return rings;
  }

  function coords () {
    var list = [];
    var item;
    var pt;
    while (pt =
           $(tuples) ||
             $(/^(,)/)) {
      if (pt === ',') {
        list.push(item);
        item = [];
      } else if (!pt.split(/\s/g).some(isNaN)) {
        if (!item) item = [];
        Array.prototype.push.apply(item, pt.split(/\s/g).map(parseFloat));
      }
      white();
    }

    if (item) list.push(item);
    else return null;

    return list.length ? list : null;
  }

  function point () {
    if (!$(/^(point(\sz)?)/i)) return null;
    white();
    if (!$(/^(\()/)) return null;
    var c = coords();
    if (!c) return null;
    white();
    if (!$(/^(\))/)) return null;
    return {
      type: 'Point',
      coordinates: c[0]
    };
  }

  function multipoint () {
    if (!$(/^(multipoint)/i)) return null;
    white();
    var newCoordsFormat = _
      .substring(_.indexOf('(') + 1, _.length - 1)
      .replace(/\(/g, '')
      .replace(/\)/g, '');
    _ = 'MULTIPOINT (' + newCoordsFormat + ')';
    var c = multicoords();
    if (!c) return null;
    white();
    return {
      type: 'MultiPoint',
      coordinates: c
    };
  }

  function multilinestring () {
    if (!$(/^(multilinestring)/i)) return null;
    white();
    var c = multicoords();
    if (!c) return null;
    white();
    return {
      type: 'MultiLineString',
      coordinates: c
    };
  }

  function linestring () {
    if (!$(/^(linestring(\sz)?)/i)) return null;
    white();
    if (!$(/^(\()/)) return null;
    var c = coords();
    if (!c) return null;
    if (!$(/^(\))/)) return null;
    return {
      type: 'LineString',
      coordinates: c
    };
  }

  function polygon () {
    if (!$(/^(polygon(\sz)?)/i)) return null;
    white();
    var c = multicoords();
    if (!c) return null;
    return {
      type: 'Polygon',
      coordinates: c
    };
  }

  function multipolygon () {
    if (!$(/^(multipolygon)/i)) return null;
    white();
    var c = multicoords();
    if (!c) return null;
    return {
      type: 'MultiPolygon',
      coordinates: c
    };
  }

  function geometrycollection () {
    var geometries = [];
    var geometry;

    if (!$(/^(geometrycollection)/i)) return null;
    white();

    if (!$(/^(\()/)) return null;
    while (geometry = root()) {
      geometries.push(geometry);
      white();
      $(/^(,)/);
      white();
    }
    if (!$(/^(\))/)) return null;

    return {
      type: 'GeometryCollection',
      geometries: geometries
    };
  }

  function root () {
    return point() ||
      linestring() ||
      polygon() ||
      multipoint() ||
      multilinestring() ||
      multipolygon() ||
      geometrycollection();
  }

  return crs(root());
}
428050
def read_token(fd):
    
    out = ''
    # Skip initial whitespace.

    while 1:
        ch = fd.read(1)
        if ch.isspace(): continue
        elif len(ch) == 0: return out
        else: break

    while not ch.isspace():
        out = out + ch
        ch = fd.read(1)
        if len(ch) == 0: break

    return out
635276
def parse_object(lexer: Lexer, is_const: bool) -> ObjectValueNode:
    """"""
    start = lexer.token
    item = cast(Callable[[Lexer], Node], partial(parse_object_field, is_const=is_const))
    return ObjectValueNode(
        fields=any_nodes(lexer, TokenKind.BRACE_L, item, TokenKind.BRACE_R),
        loc=loc(lexer, start),
    )
383167
def parse_token_contents(parser, token):
    
    bits = token.split_contents()
    tag = bits.pop(0)
    args = []
    kwargs = {}
    asvar = None
    if len(bits) >= 2 and bits[-2] == "as":
        asvar = bits[-1]
        bits = bits[:-2]
    for bit in bits:
        match = kwarg_re.match(bit)
        if not match:
            raise TemplateSyntaxError('Malformed arguments to tag "{}"'.format(tag))
        name, value = match.groups()
        if name:
            kwargs[name] = parser.compile_filter(value)
        else:
            args.append(parser.compile_filter(value))
    return {"tag": tag, "args": args, "kwargs": kwargs, "asvar": asvar}
769418
function parsePointer(pointer) {
  // Let's split pointer string by tokens' separator character.
  // Also we will reverse resulting array to simplify it's further usage.
  var tokens = pointer.split(tokenSeparator).reverse();
  // Last item in resulting array is always an empty string, we don't need it.
  tokens.pop();
  // Now tokens' array is ready to use
  return tokens;
}
1091079
function readToken(data, offset) {
        var token = '', ch = data[offset];
        while (ch !== 13 && ch !== 10) {
          if (++offset >= data.length) {
            break;
          }
          token += String.fromCharCode(ch);
          ch = data[offset];
        }
        return token;
      }
1091794
private static String readToken(Buffer buffer) {
    try {
      long tokenSize = buffer.indexOfElement(TOKEN_DELIMITERS);
      if (tokenSize == -1L) tokenSize = buffer.size();

      return tokenSize != 0L
          ? buffer.readUtf8(tokenSize)
          : null;
    } catch (EOFException e) {
      throw new AssertionError();
    }
  }
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
776908
def _check_pointers(parser):
    """"""
    from fortpy.stats.bp import check_pointers
    check_pointers(parser, args["source"], args["filter"], args["recursive"])
1019751
private void parseToken(String token) {
        int len = token.length();

        for (int i = 0; i < len; i++) {
            int chr = sr.read();
            if (chr == -1) {
                throw new IllegalArgumentException("EOF reached while reading token: " + token);
            }
            chr = Character.toLowerCase((char)chr);
            int loTokenChar = token.charAt(i);

            if (loTokenChar != chr) {
                throw new IllegalArgumentException("Expected token: " + token + " at position " + sr.getPosition());
            }
        }
    }
27793
def parse_stream(stream):
    """"""
    code = []

    for (line, col, (token, value)) in Tokenizer(stream).tokenize():
        if token == Tokenizer.STRING:
            value = '"' + value + '"'
        code.append(value)

    return code
1008723
def structparser(token):
    """"""
    m = STRUCT_PACK_RE.match(token)
    if not m:
        return [token]
    else:
        endian = m.group('endian')
        if endian is None:
            return [token]
        # Split the format string into a list of 'q', '4h' etc.
        formatlist = re.findall(STRUCT_SPLIT_RE, m.group('fmt'))
        # Now deal with mulitiplicative factors, 4h -> hhhh etc.
        fmt = ''.join([f[-1] * int(f[:-1]) if len(f) != 1 else
                       f for f in formatlist])
        if endian == '@':
            # Native endianness
            if byteorder == 'little':
                endian = '<'
            else:
                assert byteorder == 'big'
                endian = '>'
        if endian == '<':
            tokens = [REPLACEMENTS_LE[c] for c in fmt]
        else:
            assert endian == '>'
            tokens = [REPLACEMENTS_BE[c] for c in fmt]
    return tokens
831319
def parse(self):
        """"""
        expr = self.expression()
        if not isinstance(self.current_token, EndToken):
            raise InvalidTokenError("Unconsumed trailing tokens.")
        return expr