Delete data older than cycle_interval, splitting the target data
into chunks of chunk_size size.
776216
def cycle_data(self, repository, cycle_interval, chunk_size, sleep_time):
        """"""

        max_timestamp = datetime.datetime.now() - cycle_interval

        # seperate datums into chunks
        while True:
            perf_datums_to_cycle = list(self.filter(
                repository=repository,
                push_timestamp__lt=max_timestamp).values_list('id', flat=True)[:chunk_size])
            if not perf_datums_to_cycle:
                # we're done!
                break
            self.filter(id__in=perf_datums_to_cycle).delete()
            if sleep_time:
                # Allow some time for other queries to get through
                time.sleep(sleep_time)

        # also remove any signatures which are (no longer) associated with
        # a job
        for signature in PerformanceSignature.objects.filter(
                repository=repository):
            if not self.filter(signature=signature).exists():
                signature.delete()
762945
def _cleanup(self, kill, verbose):
        
        if kill:
            removed_indices = self.g.prune()

            self.nout -= len(removed_indices)

            if verbose and removed_indices:
                print('Removing %s' % removed_indices)

            for j in removed_indices:
                self.inv_map.pop(j[0])
1187954
def _file_chunks(self, data, chunk_size):
        
        for i in xrange(0, len(data), chunk_size):
            yield self.compressor(data[i:i+chunk_size])
1212182
def cleanup_storage(*args):
    """"""
    ShardedClusters().cleanup()
    ReplicaSets().cleanup()
    Servers().cleanup()
    sys.exit(0)
601212
def delete_data(self):
        
        self.clean_up()
        tools.delete_dir_rec(self._download_path)
        if self._save_state_file.exists():
            self._save_state_file.unlink()
1231563
def prune_cache(mtime=15)
      # get old, unused entries and discard from DB and filesystem
      entries = remove_old_items(mtime)
      entries.each do |key, entry|
        FileUtils.rm_f(entry[:file])
      end
    end
401081
def clear_data(self):
        
        self.log(u"Clear audio_data")
        self.__samples_capacity = 0
        self.__samples_length = 0
        self.__samples = None
1033619
def delete(self, filething=None):
        """"""

        fileobj = filething.fileobj
        data = _APEv2Data(fileobj)
        if data.start is not None and data.size is not None:
            delete_bytes(fileobj, data.end - data.start, data.start)
        self.clear()
1289161
def _free_up_space(self, size, this_rel_path=None):
        '''

        '''

        # Amount of space we are over ( bytes ) for next put
        space = self.size + size - self.maxsize

        if space <= 0:
            return

        removes = []

        for row in self.database.execute("SELECT path, size, time FROM files ORDER BY time ASC"):

            if space > 0:
                removes.append(row[0])
                space -= row[1]
            else:
                break

        for rel_path in removes:
            if rel_path != this_rel_path:
                global_logger.debug("Deleting {}".format(rel_path))
                self.remove(rel_path)
814269
def purge(self, session, checksum):
        '''
        
        '''
        C = session.query(model.Calculation).get(checksum)

        if not C:
            return 'Calculation does not exist!'

        # dataset deletion includes editing the whole dataset hierarchical tree (if any)
        if C.siblings_count:
            C_meta = session.query(model.Metadata).get(checksum)
            higher_lookup = {}
            more = C.parent
            distance = 0
            while True:
                distance += 1
                higher, more = more, []
                if not higher:
                    break
                for item in higher:
                    try:
                        higher_lookup[distance].add(item)
                    except KeyError:
                        higher_lookup[distance] = set([item])
                    if item.parent:
                        more += item.parent
            for distance, members in higher_lookup.items():
                for member in members:
                    if distance == 1:
                        member.siblings_count -= 1

                    if not member.siblings_count:
                        return 'The parent dataset contains only one (current) item, please, delete parent dataset first!'

                    member.meta_data.download_size -= C_meta.download_size
                    session.add(member)
        # low-level entry deletion deals with additional tables
        else:
            session.execute( model.delete( model.Spectra ).where( model.Spectra.checksum == checksum) )
            session.execute( model.delete( model.Electrons ).where( model.Electrons.checksum == checksum ) )
            session.execute( model.delete( model.Phonons ).where( model.Phonons.checksum == checksum ) )
            session.execute( model.delete( model.Recipinteg ).where( model.Recipinteg.checksum == checksum ) )
            session.execute( model.delete( model.Basis ).where( model.Basis.checksum == checksum ) )
            session.execute( model.delete( model.Energy ).where( model.Energy.checksum == checksum ) )
            session.execute( model.delete( model.Spacegroup ).where( model.Spacegroup.checksum == checksum ) )
            session.execute( model.delete( model.Struct_ratios ).where( model.Struct_ratios.checksum == checksum ) )
            session.execute( model.delete( model.Struct_optimisation ).where( model.Struct_optimisation.checksum == checksum ) )

            struct_ids = [ int(i[0]) for i in session.query(model.Structure.struct_id).filter(model.Structure.checksum == checksum).all() ]
            for struct_id in struct_ids:
                session.execute( model.delete( model.Atom ).where( model.Atom.struct_id == struct_id ) )
                session.execute( model.delete( model.Lattice ).where( model.Lattice.struct_id == struct_id ) )
            session.execute( model.delete( model.Structure ).where( model.Structure.checksum == checksum ) )

        # for all types of entries
        if len(C.references):
            left_references = [ int(i[0]) for i in session.query(model.Reference.reference_id).join(model.metadata_references, model.Reference.reference_id == model.metadata_references.c.reference_id).filter(model.metadata_references.c.checksum == checksum).all() ]
            session.execute( model.delete( model.metadata_references ).where( model.metadata_references.c.checksum == checksum ) )

            # remove the whole citation?
            for lc in left_references:
                if not (session.query(model.metadata_references.c.checksum).filter(model.metadata_references.c.reference_id == lc).count()):
                    session.execute( model.delete( model.Reference ).where(model.Reference.reference_id == lc) )

        # TODO rewrite with cascading
        session.execute( model.delete( model.Metadata ).where( model.Metadata.checksum == checksum ) )

        session.execute( model.delete( model.Grid ).where( model.Grid.checksum == checksum ) )
        session.execute( model.delete( model.tags ).where( model.tags.c.checksum == checksum ) )

        session.execute( model.delete( model.calcsets ).where( model.calcsets.c.children_checksum == checksum ) )
        session.execute( model.delete( model.calcsets ).where( model.calcsets.c.parent_checksum == checksum ) )
        session.execute( model.delete( model.Calculation ).where( model.Calculation.checksum == checksum ) )
        session.commit()
        # NB tables topics, codefamily, codeversion, pottype are mostly irrelevant and, if needed, should be cleaned manually
        return False
637816
def cleanup(self):
        
        date_cur_ = datetime.now()
        for seq in (self._mv_cookie, self._mv):
            for k in list(seq.keys()):
                if (date_cur_ - seq[k][1]) > timedelta(minutes=1):
                    log.debug('Cleanup: deleting entry %s', seq[k][0])
                    del seq[k]
586249
def chunk_data(data, chunkSize):
    '''
        
    '''
    chunkSize = int(chunkSize)
    return [data[i : i + chunkSize] for i in range(0, len(data), chunkSize)]
1046166
def versionable_delete(self, instance, timestamp):
        
        instance._delete_at(timestamp, using=self.using)
1214423
def trim_data(xmin, xmax, xdata, *args):
    

    # make sure it's a numpy array
    if not isinstance(xdata, _n.ndarray): xdata = _n.array(xdata)

    # make sure xmin and xmax are numbers
    if xmin is None: xmin = min(xdata)
    if xmax is None: xmax = max(xdata)

    # get all the indices satisfying the trim condition
    ns = _n.argwhere((xdata >= xmin) & (xdata <= xmax)).transpose()[0]

    # trim the xdata
    output = []
    output.append(xdata[ns])

    # trim the rest
    for a in args:
        # make sure it's a numpy array
        if not isinstance(a, _n.ndarray): a = _n.array(a)
        output.append(a[ns])

    return output
1328805
def delete_pushdata(pushdata)
      @chunks = @chunks.inject([]) do |list, chunk|
        list << chunk if chunk.pushdata != pushdata
        list
      end
      return self
    end
91443
def _cleanup_slsmod_high_data(high_data):
    '''
    
    '''
    for i in six.itervalues(high_data):
        if 'stateconf' in i:
            stateconf_data = i['stateconf'][1]
            if 'slsmod' in stateconf_data:
                stateconf_data['slsmod'] = None
967706
private void trim() {

        // Trim
        while (values.size() > limit.getMaxEntries()) {
            values.removeLast();
        }

        // Trim according to duration
        if (limit.getMaxDuration() > 0 && !values.isEmpty()) {
            long duration = limit.getMaxDuration();
            long start = values.getFirst().getTimestamp();
            while (start - values.getLast().getTimestamp() > duration) {
                values.removeLast();
            }
        }
    }