Return a chunk from a file based on the data received
503818
def serve_file(load, fnd):
    '''
    
    '''
    if 'env' in load:
        # "env" is not supported; Use "saltenv".
        load.pop('env')

    ret = {'data': '',
           'dest': ''}
    if 'path' not in load or 'loc' not in load or 'saltenv' not in load:
        return ret
    if not fnd['path']:
        return ret
    ret['dest'] = fnd['rel']
    gzip = load.get('gzip', None)
    fpath = os.path.normpath(fnd['path'])
    with salt.utils.files.fopen(fpath, 'rb') as fp_:
        fp_.seek(load['loc'])
        data = fp_.read(__opts__['file_buffer_size'])
        if gzip and data:
            data = salt.utils.gzip_util.compress(data, gzip)
            ret['gzip'] = gzip
        ret['data'] = data
    return ret
621030
def process_data(self, block):
        """"""
        if hasattr(block, 'send_destinations') and block.send_destinations:
            self.fire(events.FileProcessed(block))
            self._log_in_db(block)
            if self._sent_log_file:
                self._log_in_sent_log(block)
            self.log.info("Sent to '%s' file '%s' containing files: %s",
                          str(block.send_destinations),
                          block.processed_data_file_info.basename,
                          str([file_info.path for file_info in block.content_file_infos]))
        else:
            self.log.info("File %s wasn't sent", block.processed_data_file_info.basename)
        return block
432730
def get_chunked_content(self, chunksize=4096):
        ''''''

        # get the datastream dissemination, but return the actual http response
        r = self.obj.api.getDatastreamDissemination(self.obj.pid, self.id,
            stream=True,  asOfDateTime=self.as_of_date)
        # read and yield the response in chunks
        for chunk in r.iter_content(chunksize):
            yield chunk
1366488
def process_text(self, kind, context):
		""""""
		
		result = None
		
		while True:
			chunk = yield None
			
			if chunk is None:
				if result:
					yield result.clone(line=repr(result.line))
				
				return
			
			if not result:
				result = chunk
				continue
			
			result.line += chunk.line
1194686
def _download_chunk(self, chunk_offset, chunk_size):
        """"""
        range_id = 'bytes={0}-{1}'.format(
            chunk_offset, chunk_offset + chunk_size - 1)

        return self._blob_service.get_blob(
            container_name=self._container_name,
            blob_name=self._blob_name,
            x_ms_range=range_id)
1268493
def bytes(self, count):
        """"""

        if count < 0:
            raise ValueError

        # fast path
        if self._bits == 0:
            data = self._fileobj.read(count)
            if len(data) != count:
                raise BitReaderError("not enough data")
            return data

        return bytes(bytearray(self.bits(8) for _ in xrange(count)))
134827
def get_chunks(data, chunks=None):
    """"""

    if chunks is None:

        if hasattr(data, 'chunklen') and hasattr(data, 'shape'):
            # bcolz carray, chunk first dimension only
            return (data.chunklen,) + data.shape[1:]

        elif hasattr(data, 'chunks') and hasattr(data, 'shape') and \
                len(data.chunks) == len(data.shape):
            # h5py dataset or zarr array
            return data.chunks

        else:
            # fall back to something simple, ~4Mb chunks of first dimension
            row = np.asarray(data[0])
            chunklen = max(1, (2**22) // row.nbytes)
            if row.shape:
                chunks = (chunklen,) + row.shape
            else:
                chunks = (chunklen,)
            return chunks

    else:

        return chunks
111262
def readchunk(self):
        
        received = len(self.__buffer)
        chunk_data = EMPTY
        chunk_size = int(self.chunk_size)

        if received > 0:
            chunk_data = self.__buffer
        elif self.__position < int(self.length):
            chunk_number = int((received + self.__position) / chunk_size)
            chunk = self.__chunks.find_one({"files_id": self._id,
                                            "n": chunk_number})
            if not chunk:
                raise CorruptGridFile("no chunk #%d" % chunk_number)

            chunk_data = chunk["data"][self.__position % chunk_size:]

            if not chunk_data:
                raise CorruptGridFile("truncated chunk")

        self.__position += len(chunk_data)
        self.__buffer = EMPTY
        return chunk_data
1150932
def get_starting_chunk(filename, length=1024):
    
    # Ensure we open the file in binary mode
    with open(filename, 'rb') as f:
        chunk = f.read(length)
        return chunk
231839
def __get_chunk(self, start=0, end=None):
        '''
        
        '''
        piece = False

        if (self.__filename and os.access(self.__filename, os.R_OK)):

            fhandle = None

            try:
                fhandle = os.open(self.__filename, os.O_RDONLY)
            except OSError:
                print(("Couldn't open file %s" % (self.__filename)))
                fhandle = None

            if (fhandle):

                try:
                    sarmap = mmap.mmap(fhandle, length=0, prot=mmap.PROT_READ)
                except (TypeError, IndexError):
                    os.close(fhandle)
                    traceback.print_exc()
                    #sys.exit(-1)
                    return False

                if (not end):
                    end = sarmap.size()

                try:
                    sarmap.seek(start)
                    piece = sarmap.read(end - start)
                except:
                    traceback.print_exc()

            os.close(fhandle)

        return(piece)
259562
def read(self, chunk_size=None):
        
        if chunk_size:
            data = self.content[self.pos:self.pos + chunk_size]
            self.pos += len(data)
            return data
        else:
            return self.content
746114
async def read(self, *, decode: bool=False) -> Any:
        
        if self._at_eof:
            return b''
        data = bytearray()
        while not self._at_eof:
            data.extend((await self.read_chunk(self.chunk_size)))
        if decode:
            return self.decode(data)
        return data
464245
private void readChunk()
	throws IOException {

	String line = readLine(_in).trim();
	int length  = Integer.parseInt(line, 16);

	if (length > 0) {

	    // make sure the chunk will fit into the buffer

	    if (length > _buf.length) {
		_buf = new byte[length];
	    }

	    int bytesLeft = length;
	    int reqBytes = 0;
	    int off = 0;
	    int read = 0;

	    /*  multiple reads might be necessary to load
		the entire chunk */
	    while (bytesLeft != 0) {
		reqBytes = bytesLeft;
		read = _in.read(_buf, off, reqBytes);
		if (read == -1) break;
		bytesLeft -= read;
		off += read;
	    }

	    _max = off;
	    _index = 0;

	} else {
	    // end of data indicated
	    _eof = true;
	}

	_in.read(); // skip CR
	_in.read(); // skip LF

    }
233008
def receive_data_chunk(self, raw_data, start):
        
        self.file.write(raw_data)
        # CHANGED: This un-hangs us long enough to keep things rolling.
        eventlet.sleep(0)
1018443
def _GetChunkForReading(self, chunk):
    """"""
    offset = chunk * self._HASH_SIZE
    self.index.seek(offset)

    chunk_name = self.index.read(self._HASH_SIZE)

    try:
      return self.chunk_cache.Get(chunk_name)
    except KeyError:
      pass

    # We don't have this chunk already cached. The most common read
    # access pattern is contiguous reading so since we have to go to
    # the data store already, we read ahead to reduce round trips.
    self.index.seek(offset)
    readahead = []

    for _ in range(self._READAHEAD):
      name = self.index.read(self._HASH_SIZE)
      if name and name not in self.chunk_cache:
        readahead.append(rdf_objects.BlobID.FromBytes(name))

    self._ReadChunks(readahead)
    try:
      return self.chunk_cache.Get(chunk_name)
    except KeyError:
      raise aff4.ChunkNotFoundError("Cannot open chunk %s" % chunk)
12882
def _GetChunk(self):
    """"""

    found_ref = None
    for ref in self._blob_refs:
      if self._offset >= ref.offset and self._offset < (ref.offset + ref.size):
        found_ref = ref
        break

    if not found_ref:
      return None, None

    # If self._current_ref == found_ref, then simply return previously found
    # chunk. Otherwise, update self._current_chunk value.
    if self._current_ref != found_ref:
      self._current_ref = found_ref

      data = data_store.BLOBS.ReadBlobs([found_ref.blob_id])
      self._current_chunk = data[found_ref.blob_id]

    return self._current_chunk, self._current_ref
645741
def _handle_read_chunk(self):
        """"""
        new_data = b''
        buffer_length = len(self.read_buffer)
        try:
            while buffer_length < self.MAX_BUFFER_SIZE:
                try:
                    piece = self.recv(4096)
                except OSError as e:
                    if e.errno == errno.EAGAIN:
                        # End of the available data
                        break
                    elif e.errno == errno.EIO and new_data:
                        # Hopefully we could read an error message before the
                        # actual termination
                        break
                    else:
                        raise

                if not piece:
                    # A closed connection is indicated by signaling a read
                    # condition, and having recv() return 0.
                    break

                new_data += piece
                buffer_length += len(piece)

        finally:
            new_data = new_data.replace(b'\r', b'\n')
            self.read_buffer += new_data
        return new_data