Returns tokens list for given |pointer|. List is reversed, e.g.
'/simple/path' -> ['path', 'simple']

@param {String} pointer JSON pointer string.
@returns {Array} List of tokens.
769418
function parsePointer(pointer) {
  // Let's split pointer string by tokens' separator character.
  // Also we will reverse resulting array to simplify it's further usage.
  var tokens = pointer.split(tokenSeparator).reverse();
  // Last item in resulting array is always an empty string, we don't need it.
  tokens.pop();
  // Now tokens' array is ready to use
  return tokens;
}
1166812
function getPathTokens(path) {
	const cleanPath = path.startsWith('/') ? path.substr(1) : path;
	return cleanPath.split('/').filter((s) => s !== '');
}
67177
function(str) {
	var tokens = [] // size = str.length + 2
	var i = 0;
	var maxlen = str.length;
	TOKEN:
	while (i < maxlen) {
		var c = str.charAt(i);
		var mlo = m_tokens_start.get(c);
		// if there are multi-char tokens starting with this char, try them
		if (mlo != null) {
			for (var len = mlo; len > 1; len--) {
				if (i <= maxlen - len) {
					var tr = str.substring(i, i + len);
					if (m_tokens.contains(tr)) {
						tokens.push(tr);
						i += len;
						continue TOKEN;
					}
				}
			}
		}
		// things starting with backslash are special
		if (c == '\\' && i <= maxlen - 2) {
			if (str.charAt(i + 1) == 'u' && i <= maxlen - 6) {
				tokens.push(str.substring(i, i + 6));		// \\uxxxx
				i += 6;
			} else if (str.charAt(i + 1) == 'U' && i <= maxlen - 10) {
				tokens.push(str.substring(i, i + 10));		// \\Uxxxxxxxx
				i += 10;
			} else {
				tokens.push(str.substring(i, i + 2));		// \\x
				i += 2;
			}
			continue TOKEN;
		}
		// otherwise just take one char
		tokens.push(c.toString());
		i += 1;
	}
	return tokens;
}
79654
function parse(tokens) {
    var ast = [];

    var current = ast;

    var _line = 0;
    var _char = 0;

    var openCount = 0;
    var closeCount = 0;

    for (var i = 0; i < tokens.length; i++) {
        var token = tokens[i];

        switch (token.type) {
            case TYPES.open:
                // Every time we open a list, we drop into a sub-array
                var child = [];
                child.parent = current;
                current.push(child);
                current = child;
                openCount += 1;
                break;

            case TYPES.open_array:
            case TYPES.quote:
                var child = [];
                var quote = {"'": child };
                child.parent = current;
                current.push(quote);
                current = child;
                openCount += 1;
                break;

            case TYPES.close_array:
            case TYPES.close:
                // If no current, we probably have too many closing parens
                if (!current) _tooManyClosingParensErr(closeCount, openCount);
                // If we close a list, jump back up to the parent list
                current = current.parent;
                closeCount += 1;
                break;

            case TYPES.identifier:
            case TYPES.suffixed_number:
                current.push(token.string);
                break;

            case TYPES.number:
                current.push(Number(token.string));
                break;

            case TYPES.string:
                // We need to strip the quotes off the string entity
                var dequoted = token.string.slice(1).slice(0, token.string.length - 2);
                current.push(dequoted);
                break;

            case TYPES.json:
                try {
                    var deticked = token.string.slice(1).slice(0, token.string.length - 2);
                    current.push(JSON.parse(deticked));
                }
                catch (e) {
                    throw new Error([
                        'Runiq: Couldn\'t parse inlined JSON!',
                        '--- Error occurred at line ' + _line + ', char ' + _char
                    ].join('\n'));
                }
                break;
        }

        // Capture line numbers and columns for future use
        var lines = token.string.split(NEWLINE);
        _line += lines.length - 1;
        if (lines.length > 1) _char = lines[lines.length - 1].length;
        else _char += token.string.length;
    }

    // Raise error if we have a parentheses mismatch
    if (openCount > closeCount) _tooManyOpenParensErr(closeCount, openCount);
    if (openCount < closeCount) _tooManyClosingParensErr(closeCount, openCount);

    // For both safety and to ensure we actually have a JSON-able AST
    return JSON.parse(JSONStableStringify(ast));
}
383529
List<HostToken> parseTokenListFromJson(String json) {

        List<HostToken> hostTokens = new ArrayList<HostToken>();

        JSONParser parser = new JSONParser();
        try {
            JSONArray arr = (JSONArray) parser.parse(json);

            Iterator<?> iter = arr.iterator();
            while (iter.hasNext()) {

                Object item = iter.next();
                if (!(item instanceof JSONObject)) {
                    continue;
                }
                JSONObject jItem = (JSONObject) item;
                Long token = Long.parseLong((String) jItem.get("token"));
                String hostname = (String) jItem.get("hostname");
                String ipAddress = (String) jItem.get("ip");
                String zone = (String) jItem.get("zone");
                String datacenter = (String) jItem.get("dc");
                String portStr = (String) jItem.get("port");
                String securePortStr = (String) jItem.get("secure_port");
                String hashtag = (String) jItem.get("hashtag");
                int port = Host.DEFAULT_PORT;
                if (portStr != null) {
                    port = Integer.valueOf(portStr);
                }

                int securePort = port;
                if (securePortStr != null) {
                    securePort = Integer.valueOf(securePortStr);
                }

                Host host = new Host(hostname, ipAddress, port, securePort, zone, datacenter, Status.Up, hashtag);

                if (isLocalDatacenterHost(host)) {
                    HostToken hostToken = new HostToken(token, host);
                    hostTokens.add(hostToken);
                }
            }

        } catch (ParseException e) {
            Logger.error("Failed to parse json response: " + json, e);
            throw new RuntimeException(e);
        }

        return hostTokens;
    }
989352
function(source) {
			// transform tokens
			var tokens = this.lex(source), pos = 0, token;
			for (var i = 0, il = tokens.length; i < il; i++) {
				token = tokens[i];
				token.start = pos;
				token.end = (pos += token.value.length);
			}
			return tokens;
		}
1062738
public static List<String> tokenize_path(String path) {
        String[] tokens = path.split(SEPARATOR);
        java.util.ArrayList<String> rtn = new ArrayList<>();
        for (String str : tokens) {
            if (!str.isEmpty()) {
                rtn.add(str);
            }
        }
        return rtn;
    }
686965
private function parse($pointer)
    {
        if (!is_string($pointer)) {
            throw InvalidPointerException::invalidType(gettype($pointer));
        }

        if (!isset($pointer[0])) {
            return [];
        }

        // If the pointer is a url fragment, it needs to be url decoded.
        if ($pointer[0] === '#') {
            $pointer = urldecode(substr($pointer, 1));
        }

        // For convenience add the beginning slash if it's missing.
        if (isset($pointer[0]) && $pointer[0] !== '/') {
            $pointer = '/' . $pointer;
        }

        $pointer = array_slice(explode('/', $pointer), 1);
        return str_replace(['~1', '~0'], ['/', '~'], $pointer);
    }
25161
function getWordToken(str) {
    var word = str.split(/\s/g)[0];
    return {text: word, type: 'WORD', length: word.length};
}
128945
public static List<String> tokenize_path(String path) {
        String[] toks = path.split(SEPERATOR);
        ArrayList<String> rtn = new ArrayList<String>();
        for (String str : toks) {
            if (!str.isEmpty()) {
                rtn.add(str);
            }
        }
        return rtn;
    }
172299
function fetchObjectByPointer(pointer) {
  const collection = getCollection(pointer.className);
  const storedItem = collection[pointer.objectId];

  if (storedItem === undefined) {
    return undefined;
  }

  return Object.assign(
    { __type: 'Object', className: pointer.className },
    _.cloneDeep(storedItem)
  );
}
84411
function(ptr) {
			var i,
				l,
				cl = this.children;

			if (cl) {
				if (typeof ptr === "string") {
					for (i = 0, l = cl.length; i < l; i++) {
						if (cl[i].key === ptr) {
							return cl[i];
						}
					}
				} else if (typeof ptr === "number") {
					return this.children[ptr];
				} else if (ptr.parent === this) {
					return ptr;
				}
			}
			return null;
		}
487702
function tokensId(tokens, prepend) {
    if (!prepend) return null;

    return _.reduce(tokens, function(prev, token) {
        var tokId = [
            token.index,
            token.offset,
            token.value
        ].join('-');

        return [prev, tokId].join(':');
    }, prepend);
}
7611
function pathExpressions (token, tokens) {
  const expressions = []
  token = tokens.shift()
  while (true) {
    if (!token) {
      throw new Error(`A path expression should end with ")"`)
    }
    if (token === ')') {
      break
    }
    expressions.push(pathExpression(token, tokens))
    token = tokens.shift()
  }
  return { _type: pathExpressions.name, expressions }
}
1154428
public String[] getAttributes(String key) {
        if (token != null)
            return token.getAttributes(key);
        else
            return null;
    }
107638
function fetchTokensData (tokenRegContract, tokenIndexes) {
  return (dispatch, getState) => {
    const { api, tokens } = getState();
    const allTokens = Object.values(tokens);

    const tokensIndexesMap = allTokens
      .reduce((map, token) => {
        map[token.index] = token;
        return map;
      }, {});

    const fetchedTokenIndexes = allTokens
      .filter((token) => token.fetched)
      .map((token) => token.index);

    const fullIndexes = [];
    const partialIndexes = [];

    tokenIndexes.forEach((tokenIndex) => {
      if (fetchedTokenIndexes.includes(tokenIndex)) {
        partialIndexes.push(tokenIndex);
      } else {
        fullIndexes.push(tokenIndex);
      }
    });

    log.debug('need to fully fetch', fullIndexes);
    log.debug('need to partially fetch', partialIndexes);

    const fullPromise = fetchTokensInfo(api, tokenRegContract, fullIndexes);
    const partialPromise = fetchTokensImages(api, tokenRegContract, partialIndexes)
      .then((imagesResult) => {
        return imagesResult.map((image, index) => {
          const tokenIndex = partialIndexes[index];
          const token = tokensIndexesMap[tokenIndex];

          return { ...token, image };
        });
      });

    return Promise.all([ fullPromise, partialPromise ])
      .then(([ fullResults, partialResults ]) => {
        log.debug('fetched', { fullResults, partialResults });

        return []
          .concat(fullResults, partialResults)
          .filter(({ address }) => !/0x0*$/.test(address))
          .reduce((tokens, token) => {
            const { id } = token;

            tokens[id] = token;
            return tokens;
          }, {});
      })
      .then((tokens) => {
        dispatch(setTokens(tokens));
      });
  };
}
845202
def _list_po_to_dict(tokens) -> ListAbundance:
    
    func = tokens[FUNCTION]
    dsl = FUNC_TO_LIST_DSL[func]

    members = [parse_result_to_dsl(token) for token in tokens[MEMBERS]]

    return dsl(members)