Read a page containing dictionary data.

    Consumes data using the plain encoding and returns an array of values.
348332
def _read_dictionary_page(file_obj, schema_helper, page_header, column_metadata):
    
    raw_bytes = _read_page(file_obj, page_header, column_metadata)
    io_obj = io.BytesIO(raw_bytes)
    values = encoding.read_plain(
        io_obj,
        column_metadata.type,
        page_header.dictionary_page_header.num_values
    )
    # convert the values once, if the dictionary is associated with a converted_type.
    schema_element = schema_helper.schema_element(column_metadata.path_in_schema[-1])
    return convert_column(values, schema_element) if schema_element.converted_type is not None else values
35782
def _get_data(url):
    """"""
    if urllib_parse.urlparse(url).scheme in ('http', 'https'):
        resp = urllib_request.urlopen(url)
        encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            data = data.decode('utf-8')
        elif encoding == 'gzip':
            data = BytesIO(data)
            data = gzip.GzipFile(fileobj=data).read().decode('utf-8')
        else:
            raise RuntimeError('unknown encoding')
    else:
        with codecs.open(url, mode='r', encoding='utf-8') as fid:
            data = fid.read()

    return data
196629
def decode_response(client_message, to_object=None):
    
    parameters = dict(read_count=None, items=None)
    parameters['read_count'] = client_message.read_int()
    items_size = client_message.read_int()
    items = []
    for _ in range(0, items_size):
        items_item = client_message.read_data()
        items.append(items_item)
    parameters['items'] = ImmutableLazyDataList(items, to_object)
    return parameters
652595
def read(self, url):
        """"""
        params = self._split_url(url)
        output_stream = io.BytesIO()
        block_blob_service = self._block_blob_service(
            account_name=params["account"], sas_token=params["sas_token"]
        )

        block_blob_service.get_blob_to_stream(
            container_name=params["container"], blob_name=params["blob"], stream=output_stream
        )

        output_stream.seek(0)
        return [line.decode("utf-8") for line in output_stream]
836918
def load_slice(self, state, start, end):
        
        items = [ ]
        if start > self._page_addr + self._page_size or end < self._page_addr:
            l.warning("Calling load_slice on the wrong page.")
            return items

        for addr in range(max(start, self._page_addr), min(end, self._page_addr + self._page_size)):
            i = addr - self._page_addr
            mo = self._storage[i]
            if mo is None:
                mo = self._sinkhole
            if mo is not None and (not items or items[-1][1] is not mo):
                items.append((addr, mo))
        return items
341200
def load(data):
    

    if hasattr(data, 'read') and callable(data.read):
      contents = data.read()
    elif isinstance(data, six.string_types):
      contents = data
    else:
      raise TypeError('Can only process data from a string or a readable object, given: %s' % data)

    return Properties._parse(contents.splitlines())
1015968
def read(self):
        
        with closing(self.open()) as f:
            data = f.read()
        return data.decode("utf-8") or None
251918
def read_data_page(file_obj, schema_helper, page_header, column_metadata,
                   dictionary):
    
    daph = page_header.data_page_header
    raw_bytes = _read_page(file_obj, page_header, column_metadata)
    io_obj = io.BytesIO(raw_bytes)
    vals = []
    debug_logging = logger.isEnabledFor(logging.DEBUG)

    if debug_logging:
        logger.debug("  definition_level_encoding: %s",
                     _get_name(parquet_thrift.Encoding, daph.definition_level_encoding))
        logger.debug("  repetition_level_encoding: %s",
                     _get_name(parquet_thrift.Encoding, daph.repetition_level_encoding))
        logger.debug("  encoding: %s", _get_name(parquet_thrift.Encoding, daph.encoding))

    # definition levels are skipped if data is required.
    definition_levels = None
    num_nulls = 0
    max_definition_level = -1
    if not schema_helper.is_required(column_metadata.path_in_schema[-1]):
        max_definition_level = schema_helper.max_definition_level(
            column_metadata.path_in_schema)
        bit_width = encoding.width_from_max_int(max_definition_level)
        if debug_logging:
            logger.debug("  max def level: %s   bit_width: %s",
                         max_definition_level, bit_width)
        if bit_width == 0:
            definition_levels = [0] * daph.num_values
        else:
            definition_levels = _read_data(io_obj,
                                           daph.definition_level_encoding,
                                           daph.num_values,
                                           bit_width)[:daph.num_values]

        # any thing that isn't at max definition level is a null.
        num_nulls = len(definition_levels) - definition_levels.count(max_definition_level)
        if debug_logging:
            logger.debug("  Definition levels: %s", len(definition_levels))

    # repetition levels are skipped if data is at the first level.
    repetition_levels = None  # pylint: disable=unused-variable
    if len(column_metadata.path_in_schema) > 1:
        max_repetition_level = schema_helper.max_repetition_level(
            column_metadata.path_in_schema)
        bit_width = encoding.width_from_max_int(max_repetition_level)
        repetition_levels = _read_data(io_obj,
                                       daph.repetition_level_encoding,
                                       daph.num_values,
                                       bit_width)

    # NOTE: The repetition levels aren't yet used.
    if daph.encoding == parquet_thrift.Encoding.PLAIN:
        read_values = encoding.read_plain(io_obj, column_metadata.type, daph.num_values - num_nulls)
        schema_element = schema_helper.schema_element(column_metadata.path_in_schema[-1])
        read_values = convert_column(read_values, schema_element) \
            if schema_element.converted_type is not None else read_values
        if definition_levels:
            itr = iter(read_values)
            vals.extend([next(itr) if level == max_definition_level else None for level in definition_levels])
        else:
            vals.extend(read_values)
        if debug_logging:
            logger.debug("  Values: %s, nulls: %s", len(vals), num_nulls)

    elif daph.encoding == parquet_thrift.Encoding.PLAIN_DICTIONARY:
        # bit_width is stored as single byte.
        bit_width = struct.unpack(b"<B", io_obj.read(1))[0]
        if debug_logging:
            logger.debug("bit_width: %d", bit_width)

        dict_values_bytes = io_obj.read()
        dict_values_io_obj = io.BytesIO(dict_values_bytes)
        # read_values stores the bit-packed values. If there are definition levels and the data contains nulls,
        # the size of read_values will be less than daph.num_values
        read_values = []
        while dict_values_io_obj.tell() < len(dict_values_bytes):
            read_values.extend(encoding.read_rle_bit_packed_hybrid(
                dict_values_io_obj, bit_width, len(dict_values_bytes)))

        if definition_levels:
            itr = iter(read_values)
            # add the nulls into a new array, values, but using the definition_levels data.
            values = [dictionary[next(itr)] if level == max_definition_level else None for level in definition_levels]
        else:
            values = [dictionary[v] for v in read_values]

        # there can be extra values on the end of the array because the last bit-packed chunk may be zero-filled.
        if len(values) > daph.num_values:
            values = values[0: daph.num_values]
        vals.extend(values)

        if debug_logging:
            logger.debug("  Read %s values using PLAIN_DICTIONARY encoding and definition levels show %s nulls",
                         len(vals), num_nulls)

    else:
        raise ParquetFormatException("Unsupported encoding: %s",
                                     _get_name(parquet_thrift.Encoding, daph.encoding))
    return vals
221834
def _read_all_values(self):
        """"""

        pos = 8

        # cache variables to local ones and prevent attributes lookup
        # on every loop iteration
        used = self._used
        data = self._m
        unpack_from = struct.unpack_from

        while pos < used:
            encoded_len = _unpack_integer(data, pos)[0]
            # check we are not reading beyond bounds
            if encoded_len + pos > used:
                msg = 'Read beyond file size detected, %s is corrupted.'
                raise RuntimeError(msg % self._fname)
            pos += 4
            encoded = unpack_from(('%ss' % encoded_len).encode(), data, pos)[0]
            padded_len = encoded_len + (8 - (encoded_len + 4) % 8)
            pos += padded_len
            value = _unpack_double(data, pos)[0]
            yield encoded.decode('utf-8'), value, pos
            pos += 8
319712
def read_dict(file_name, clear_none=False, encoding='utf-8'):
    
    with open(file_name, 'rb') as f:
        data = f.read()

    if encoding is not None:
        data = data.decode(encoding)

    line_list = data.splitlines()
    data = []
    i = 0
    for line in line_list:
        i += 1
        try:
            line = force_text(line).strip()
            data.append(line)
        except:
            print('read error line %s' % i)
    if clear_none:
        data = [t for t in data if t != '']
    data = deque(data)
    return data
1365236
def _decode_list(data):
    
    if six.PY3:
        return data
    rv = []
    for item in data:
        if not isinstance(item, text_type):
            item = item.encode('utf-8')
        elif isinstance(item, list):
            item = _decode_list(item)
        rv.append(item)
    return rv
755905
def read_plain_byte_array(file_obj, count):
    """"""
    return [file_obj.read(struct.unpack(b"<i", file_obj.read(4))[0]) for i in range(count)]
11617
def parse_json(self, page):
        ''''''
        if not isinstance(page, basestring):
            page = util.decode_page(page)
            
        self.doc = json.loads(page)
        results = self.doc.get(self.result_name, [])
    
        if not results:
            self.check_status(self.doc.get('status'))
            return None
            
            
        return results
850567
def read_string_array(self, key, embedded=True):
        
        data = None
        if key is not None:
            key_type = self.variable_type(key)
            data = self.db.read(key.strip())
            if embedded:
                data = self.read_embedded(data, key_type)
            if data is not None:
                try:
                    data = json.loads(data, object_pairs_hook=OrderedDict)
                except ValueError as e:
                    err = u'Failed loading JSON data ({}). Error: ({})'.format(data, e)
                    self.tcex.log.error(err)
                    self.tcex.message_tc(err)
                    self.tcex.exit(1)
        else:
            self.tcex.log.warning(u'The key field was None.')
        return data
904431
def urlread(url, encoding='utf8'):
    
    try:
        from urllib.request import urlopen
    except ImportError:
        from urllib2 import urlopen
    response = urlopen(url)
    content = response.read()
    content = content.decode(encoding)
    return content
757314
def read(self, raw=False, perform_traversal=True):
        
        if not raw:
            response_body = decode(self.data, self.mimetype(), encoding=self.encoding())
            if perform_traversal and self.traversal is not None:
                return traverse(response_body, *self.traversal)
            return response_body
        else:
            return self.data
590789
def _read(self, ddfile):
        
        import csv

        d = {}
        with open(ddfile) as f:
            reader = csv.DictReader(f)
            for row in reader:
                key = row['name']
                if key.startswith('#'):
                    key = key[1:]
                    deprecated = True
                else:
                    deprecated = False
                value = d.setdefault(key, {})
                value[self._get_value(row)] = deprecated
        return d