Roll window worth of data up to position zero.
        Save the effort of having to expensively roll at each iteration
356640
def _roll_data(self):
        

        self.buffer.values[:, :self._window, :] = \
            self.buffer.values[:, -self._window:, :]
        self.date_buf[:self._window] = self.date_buf[-self._window:]
        self._pos = self._window
348612
def rolling_window(a, axis, window, center, fill_value):
    
    orig_shape = a.shape
    if axis < 0:
        axis = a.ndim + axis
    depth = {d: 0 for d in range(a.ndim)}
    depth[axis] = int(window / 2)
    # For evenly sized window, we need to crop the first point of each block.
    offset = 1 if window % 2 == 0 else 0

    if depth[axis] > min(a.chunks[axis]):
        raise ValueError(
            "For window size %d, every chunk should be larger than %d, "
            "but the smallest chunk size is %d. Rechunk your array\n"
            "with a larger chunk size or a chunk size that\n"
            "more evenly divides the shape of your array." %
            (window, depth[axis], min(a.chunks[axis])))

    # Although dask.overlap pads values to boundaries of the array,
    # the size of the generated array is smaller than what we want
    # if center == False.
    if center:
        start = int(window / 2)  # 10 -> 5,  9 -> 4
        end = window - 1 - start
    else:
        start, end = window - 1, 0
    pad_size = max(start, end) + offset - depth[axis]
    drop_size = 0
    # pad_size becomes more than 0 when the overlapped array is smaller than
    # needed. In this case, we need to enlarge the original array by padding
    # before overlapping.
    if pad_size > 0:
        if pad_size < depth[axis]:
            # overlapping requires each chunk larger than depth. If pad_size is
            # smaller than the depth, we enlarge this and truncate it later.
            drop_size = depth[axis] - pad_size
            pad_size = depth[axis]
        shape = list(a.shape)
        shape[axis] = pad_size
        chunks = list(a.chunks)
        chunks[axis] = (pad_size, )
        fill_array = da.full(shape, fill_value, dtype=a.dtype, chunks=chunks)
        a = da.concatenate([fill_array, a], axis=axis)

    boundary = {d: fill_value for d in range(a.ndim)}

    # create overlap arrays
    ag = overlap(a, depth=depth, boundary=boundary)

    # apply rolling func
    def func(x, window, axis=-1):
        x = np.asarray(x)
        rolling = nputils._rolling_window(x, window, axis)
        return rolling[(slice(None), ) * axis + (slice(offset, None), )]

    chunks = list(a.chunks)
    chunks.append(window)
    out = ag.map_blocks(func, dtype=a.dtype, new_axis=a.ndim, chunks=chunks,
                        window=window, axis=axis)

    # crop boundary.
    index = (slice(None),) * axis + (slice(drop_size,
                                           drop_size + orig_shape[axis]), )
    return out[index]
75010
def generate_data(self, data_dir, tmp_dir=None, task_id=-1):
    """"""
    if not self._rollouts_by_epoch_and_split[self.current_epoch]:
      # Data not loaded from disk.
      self._split_current_epoch()

    rollouts_by_split = self._rollouts_by_epoch_and_split[self.current_epoch]
    splits_and_paths = self.splits_and_paths(data_dir)

    for (split, paths) in splits_and_paths:
      rollouts = rollouts_by_split[split]
      num_frames = self._calc_num_frames(rollouts)
      shard_size = num_frames // len(paths)

      frame_gen = self._generate_frames(rollouts)
      for (path_index, path) in enumerate(paths):
        limit = shard_size
        # Put the remainder in the last shard to preserve the ordering.
        if path_index == len(paths) - 1:
          limit = None
        generator_utils.generate_files(
            itertools.islice(frame_gen, limit), [path],
            cycle_every_n=float("inf")
        )
89361
def prange(ts, **kwargs):
    '''
    '''
    mi = ts.rollmin(**kwargs)
    ma = ts.rollmax(**kwargs)
    return (ts - mi)/(ma - mi)
9420
def flush(self):
        
        i = 0
        while self._frame_data.is_dirty and i < 10:
            i += 1
            time.sleep(0.1)
1275201
def iter(self, count=0):
        '''
        '''
        while True:
            yield super(FuncRoll, self).roll(count, self._func)
511662
def iterchunks(data, chunksize):
    
    offt = 0
    while offt < len(data):
        yield data[offt:offt+chunksize]
        offt += chunksize
177798
def dask_rolling_wrapper(moving_func, a, window, min_count=None, axis=-1):
    ''''''
    dtype, fill_value = dtypes.maybe_promote(a.dtype)
    a = a.astype(dtype)
    # inputs for overlap
    if axis < 0:
        axis = a.ndim + axis
    depth = {d: 0 for d in range(a.ndim)}
    depth[axis] = (window + 1) // 2
    boundary = {d: fill_value for d in range(a.ndim)}
    # Create overlap array.
    ag = overlap(a, depth=depth, boundary=boundary)
    # apply rolling func
    out = ag.map_blocks(moving_func, window, min_count=min_count,
                        axis=axis, dtype=a.dtype)
    # trim array
    result = trim_internal(out, depth)
    return result
319197
def _store_unsorted_results(self, smooth, residual):
        """"""
        if self._original_index_of_xvalue:
            # data was sorted. Unsort it here.
            self.smooth_result = numpy.zeros(len(self.y))
            self.cross_validated_residual = numpy.zeros(len(residual))
            original_x = numpy.zeros(len(self.y))
            for i, (xval, smooth_val, residual_val) in enumerate(zip(self.x, smooth, residual)):
                original_index = self._original_index_of_xvalue[i]
                original_x[original_index] = xval
                self.smooth_result[original_index] = smooth_val
                self.cross_validated_residual[original_index] = residual_val
                self.x = original_x
        else:
            # no sorting was done. just apply results
            self.smooth_result = smooth
            self.cross_validated_residual = residual
1220050
def load(self, dtype_out_time, dtype_out_vert=False, region=False,
             plot_units=False, mask_unphysical=False):
        """"""
        msg = ("Loading data from disk for object={0}, dtype_out_time={1}, "
               "dtype_out_vert={2}, and region="
               "{3}".format(self, dtype_out_time, dtype_out_vert, region))
        logging.info(msg + ' ({})'.format(ctime()))
        # Grab from the object if its there.
        try:
            data = self.data_out[dtype_out_time]
        except (AttributeError, KeyError):
            # Otherwise get from disk.  Try scratch first, then archive.
            try:
                data = self._load_from_disk(dtype_out_time, dtype_out_vert,
                                            region=region)
            except IOError:
                data = self._load_from_tar(dtype_out_time, dtype_out_vert)
        # Copy the array to self.data_out for ease of future access.
        self._update_data_out(data, dtype_out_time)
        # Apply desired plotting/cleanup methods.
        if mask_unphysical:
            data = self.var.mask_unphysical(data)
        if plot_units:
            data = self.var.to_plot_units(data, dtype_vert=dtype_out_vert)
        return data
572534
def _rolling_window(a, window):
    
    # https://stackoverflow.com/a/6811241
    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
    strides = a.strides + (a.strides[-1],)
    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
1164716
def collect_frames_for_random_starts(
    storage_env, stacked_env, agent, frame_stack_size, random_starts_step_limit,
    log_every_steps=None
):
  """"""
  del frame_stack_size
  storage_env.start_new_epoch(0)
  tf.logging.info(
      "Collecting %d frames for random starts.", random_starts_step_limit
  )
  rl_utils.run_rollouts(
      stacked_env, agent, stacked_env.reset(),
      step_limit=random_starts_step_limit,
      many_rollouts_from_each_env=True,
      log_every_steps=log_every_steps,
  )
  # Save unfinished rollouts to history.
  stacked_env.reset()
1111077
def rollforward(self):
        
        if self.contents['rollforward'] is None:
            raise ValueError("No operation to roll forward on refpkg")
        new_log_message = self.contents['rollforward'][0]
        new_contents = self.contents['rollforward'][1]
        new_contents['log'] = [new_log_message] + self.contents.pop('log')
        self.contents['rollforward'] = None
        new_contents['rollback'] = copy.deepcopy(self.contents)
        new_contents['rollback'].pop('rollforward')
        self.contents = new_contents
        self._sync_to_disk()
1031604
def set_data(self, data):
		""
		if data is None:
			self.data_size = 0
			self.data = None
			return
		self.data_size = len(data)
		# create a string buffer so that null bytes aren't interpreted
		#  as the end of the string
		self.data = ctypes.cast(ctypes.create_string_buffer(data), ctypes.c_void_p)
873087
private void rolloverLogTask()
  {
    try {
      if (_isInit) {
        flush();
      }
    } catch (Exception e) {
      log.log(Level.WARNING, e.toString(), e);
    }

    _isRollingOver = true;
    
    try {
      if (! _isInit)
        return;
      
      Path savedPath = null;

      long now = CurrentTime.currentTime();

      long lastPeriodEnd = _nextPeriodEnd;

      _nextPeriodEnd = nextRolloverTime(now);

      Path path = getPath();

      synchronized (_logLock) {
        flushTempStream();

        long length = Files.size(path);
        
        if (lastPeriodEnd <= now && lastPeriodEnd > 0) {
          closeLogStream();

          savedPath = getSavedPath(lastPeriodEnd - 1);
        }
        else if (path != null && getRolloverSize() <= length) {
          closeLogStream();

          savedPath = getSavedPath(now);
        }
      }
      
      // archiving of path is outside of the synchronized block to
      // avoid freezing during archive
      if (savedPath != null) {
        movePathToArchive(savedPath);
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      synchronized (_logLock) {
        _isRollingOver = false;
        flushTempStream();
      }
      
      _rolloverListener.requeue(_rolloverAlarm);
    }
  }
877454
def limit(self, n):
    
    data = self._data
    self._data = (next(data) for _ in xrange(int(round(n))))
    return self
198603
def rolling_window(array, axis, window, center, fill_value):
    
    if isinstance(array, dask_array_type):
        return dask_array_ops.rolling_window(
            array, axis, window, center, fill_value)
    else:  # np.ndarray
        return nputils.rolling_window(
            array, axis, window, center, fill_value)