Parse and yield array items from the token stream.
846
def array_items(self, number_type, *, number_suffix=''):
        """"""
        for token in self.collect_tokens_until('CLOSE_BRACKET'):
            is_number = token.type == 'NUMBER'
            value = token.value.lower()
            if not (is_number and value.endswith(number_suffix)):
                raise self.error(f'Invalid {number_type} array element '
                                 f'{token.value!r}')
            yield int(value.replace(number_suffix, ''))
99250
def __get_tokens(self, row):
        """"""
        row_tokenizer = RowTokenizer(row, self.config)

        line = row_tokenizer.next()
        while line:
            # print("RETURN line", line[0:100])
            yield line
            line = row_tokenizer.next()
247897
def parse(self, parser):
        ''''''
        # line number of token that started the tag
        lineno = next(parser.stream).lineno

        # template context
        context = nodes.ContextReference()

        # parse keyword arguments
        kwargs = []

        while parser.stream.look().type == lexer.TOKEN_ASSIGN:
            key = parser.stream.expect(lexer.TOKEN_NAME)
            next(parser.stream)
            kwargs.append(
                nodes.Keyword(key.value, parser.parse_expression()),
            )
            parser.stream.skip_if('comma')
        # parse content of the activeurl block up to endactiveurl
        body = parser.parse_statements(['name:endactiveurl'], drop_needle=True)

        args = [context]

        call_method = self.call_method(
            'render_tag',
            args=args,
            kwargs=kwargs,
        )

        return nodes.CallBlock(call_method, [], [], body).set_lineno(lineno)
643765
def get_array_indices(self):
        """"""

        for token in self.tokens:
            if isinstance(token, SquareBrackets):
                # Use [1:-1] index to discard the square brackets
                yield token.tokens[1:-1]
1299592
def read_content(path: str, limit: Optional[int] = None) -> Iterator[List[str]]:
    
    with smart_open(path) as indata:
        for i, line in enumerate(indata):
            if limit is not None and i == limit:
                break
            yield list(get_tokens(line))
292134
function array(node, parent, scope) {
  var uid = scope.generateUidIdentifierBasedOnNode(parent);

  var container = util.template("array-comprehension-container", {
    KEY: uid
  });
  container.callee.shadow = true;

  var block = container.callee.body;
  var body = block.body;

  if (_traversal2["default"].hasType(node, scope, "YieldExpression", t.FUNCTION_TYPES)) {
    container.callee.generator = true;
    container = t.yieldExpression(container, true);
  }

  var returnStatement = body.pop();

  body.push(_helpersBuildComprehension2["default"](node, function () {
    return util.template("array-push", {
      STATEMENT: node.body,
      KEY: uid
    }, true);
  }));
  body.push(returnStatement);

  return container;
}
324856
def get_tokens(tu, extent):
        
        tokens_memory = POINTER(Token)()
        tokens_count = c_uint()

        conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
                byref(tokens_count))

        count = int(tokens_count.value)

        # If we get no tokens, no memory was allocated. Be sure not to return
        # anything and potentially call a destructor on nothing.
        if count < 1:
            return

        tokens_array = cast(tokens_memory, POINTER(Token * count)).contents

        token_group = TokenGroup(tu, tokens_memory, tokens_count)

        for i in xrange(0, count):
            token = Token()
            token.int_data = tokens_array[i].int_data
            token.ptr_data = tokens_array[i].ptr_data
            token._tu = tu
            token._group = token_group

            yield token
214068
def parse(self, stream):
        """"""

        items = OrderedDict()
        for i, line in enumerate(stream):
            line = line.strip()
            if not line or line[0] in ["#", ";", "["] or line.startswith("---"):
                continue
            white_space = "\\s*"
            key = "(?P<key>[^:=;#\s]+?)"
            value = white_space+"[:=\s]"+white_space+"(?P<value>.+?)"
            comment = white_space+"(?P<comment>\\s[;#].*)?"

            key_only_match = re.match("^" + key + comment + "$", line)
            if key_only_match:
                key = key_only_match.group("key")
                items[key] = "true"
                continue

            key_value_match = re.match("^"+key+value+comment+"$", line)
            if key_value_match:
                key = key_value_match.group("key")
                value = key_value_match.group("value")

                if value.startswith("[") and value.endswith("]"):
                    # handle special case of lists
                    value = [elem.strip() for elem in value[1:-1].split(",")]

                items[key] = value
                continue

            raise ConfigFileParserException("Unexpected line %s in %s: %s" % (i,
                getattr(stream, 'name', 'stream'), line))
        return items
526157
Stream<Token> tokenize(String input) {
        List<Token> tokens = new ArrayList<Token>();
        int tokenPos = 0;
        while (!input.isEmpty()) {
            boolean matched = false;
            for (Token.Type tokenType : Token.Type.values()) {
                Matcher matcher = tokenType.pattern.matcher(input);
                if (matcher.find()) {
                    matched = true;
                    input = matcher.replaceFirst("");
                    if (tokenType != Token.Type.WHITESPACE) {
                        tokens.add(new Token(
                            tokenType,
                            matcher.group(),
                            tokenPos
                        ));
                    }
                    tokenPos += matcher.end();
                    break;
                }
            }
            if (!matched) {
                throw new LexerException(input);
            }
        }
        tokens.add(new Token(Token.Type.EOI, null, tokenPos));
        return new Stream<Token>(tokens.toArray(new Token[tokens.size()]));
    }
997294
def consume(self, kind):
        """"""
        next_token = self.stream.move()
        if next_token.kind != kind:
            raise UnexpectedTokenError(token=next_token, expected_kind=kind)
444686
def get_tokens(line: str) -> Iterator[str]:
    
    for token in line.rstrip().split():
        if len(token) > 0:
            yield token
949876
def read_stream (stream):
    """"""

    section = None
    key = None
    data = None

    for fullline in stream:
        line = fullline.split ('#', 1)[0]

        m = sectionre.match (line)
        if m is not None:
            # New section
            if section is not None:
                if key is not None:
                    section.set_one (key, data.strip ().decode ('utf8'))
                    key = data = None
                yield section

            section = Holder ()
            section.section = m.group (1)
            continue

        if len (line.strip ()) == 0:
            if key is not None:
                section.set_one (key, data.strip ().decode ('utf8'))
                key = data = None
            continue

        m = escre.match (fullline)
        if m is not None:
            if section is None:
                raise InifileError ('key seen without section!')
            if key is not None:
                section.set_one (key, data.strip ().decode ('utf8'))
            key = m.group (1)
            data = m.group (2).replace (r'\"', '"').replace (r'\n', '\n').replace (r'\\', '\\')
            section.set_one (key, data.decode ('utf8'))
            key = data = None
            continue

        m = keyre.match (line)
        if m is not None:
            if section is None:
                raise InifileError ('key seen without section!')
            if key is not None:
                section.set_one (key, data.strip ().decode ('utf8'))
            key = m.group (1)
            data = m.group (2)
            if not len (data):
                data = ' '
            elif not data[-1].isspace ():
                data += ' '
            continue

        if line[0].isspace () and key is not None:
            data += line.strip () + ' '
            continue

        raise InifileError ('unparsable line: ' + line[:-1])

    if section is not None:
        if key is not None:
            section.set_one (key, data.strip ().decode ('utf8'))
        yield section
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
375997
def _stream_helper(self, response, decode=False):
        """"""

        if response.raw._fp.chunked:
            if decode:
                for chunk in json_stream(self._stream_helper(response, False)):
                    yield chunk
            else:
                reader = response.raw
                while not reader.closed:
                    # this read call will block until we get a chunk
                    data = reader.read(1)
                    if not data:
                        break
                    if reader._fp.chunk_left:
                        data += reader.read(reader._fp.chunk_left)
                    yield data
        else:
            # Response isn't chunked, meaning we probably
            # encountered an error immediately
            yield self._result(response, json=decode)
1166847
public List<Object> parseArray() throws IOException {
		Type t = lexer.readNextToken();
		if (t != Type.START_ARRAY) {
			throw new IOException("Unexpected token: " + t);
		}
		
		return parseArrayInternal();
	}
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
22466
async def _stream_helper(self, response, decode=False):
        """"""

        if decode:
            async for chunk in json_stream(
                    self._stream_helper(response, False)):
                yield chunk
        else:
            async with response.context as res:
                reader = res.content
                while True:
                    # this read call will block until we get a chunk
                    data = await reader.read(1)
                    if not data:
                        break
                    data += await reader.read(io.DEFAULT_BUFFER_SIZE)
                    yield data