dump already partitioned data into disks.
774393
def _spill(self):
        
        global MemoryBytesSpilled, DiskBytesSpilled
        path = self._get_spill_dir(self.spills)
        if not os.path.exists(path):
            os.makedirs(path)

        used_memory = get_used_memory()
        if not self.pdata:
            # The data has not been partitioned, it will iterator the
            # data once, write them into different files, has no
            # additional memory. It only called when the memory goes
            # above limit at the first time.

            # open all the files for writing
            streams = [open(os.path.join(path, str(i)), 'wb')
                       for i in range(self.partitions)]

            # If the number of keys is small, then the overhead of sort is small
            # sort them before dumping into disks
            self._sorted = len(self.data) < self.SORT_KEY_LIMIT
            if self._sorted:
                self.serializer = self.flattened_serializer()
                for k in sorted(self.data.keys()):
                    h = self._partition(k)
                    self.serializer.dump_stream([(k, self.data[k])], streams[h])
            else:
                for k, v in self.data.items():
                    h = self._partition(k)
                    self.serializer.dump_stream([(k, v)], streams[h])

            for s in streams:
                DiskBytesSpilled += s.tell()
                s.close()

            self.data.clear()
            # self.pdata is cached in `mergeValues` and `mergeCombiners`
            self.pdata.extend([{} for i in range(self.partitions)])

        else:
            for i in range(self.partitions):
                p = os.path.join(path, str(i))
                with open(p, "wb") as f:
                    # dump items in batch
                    if self._sorted:
                        # sort by key only (stable)
                        sorted_items = sorted(self.pdata[i].items(), key=operator.itemgetter(0))
                        self.serializer.dump_stream(sorted_items, f)
                    else:
                        self.serializer.dump_stream(self.pdata[i].items(), f)
                self.pdata[i].clear()
                DiskBytesSpilled += os.path.getsize(p)

        self.spills += 1
        gc.collect()  # release the memory as much as possible
        MemoryBytesSpilled += max(used_memory - get_used_memory(), 0) << 20
361360
def __alloc_raw_data(self, initial_values=None):
        """"""
        #allocate c_ubyte storage
        if self.__raw_data == None: #first time only, create storage
            raw_data_type = c_ubyte * self.__raw_report_size
            self.__raw_data = raw_data_type()
        elif initial_values == self.__raw_data:
            # already
            return
        else:
            #initialize
            ctypes.memset(self.__raw_data, 0, len(self.__raw_data))
        if initial_values:
            for index in range(len(initial_values)):
                self.__raw_data[index] = initial_values[index]
454741
def _backup_pb_tqdm(self, dirs):
        """"""
        with ZipFile(self.zip_filename, 'w') as backup_zip:
            for path in tqdm(dirs, desc='Writing Zip Files', total=len(dirs)):
                backup_zip.write(path, path[len(self.source):len(path)])
1330996
def dump(device, args=None):
    '''
    
    '''
    cmd = 'dumpe2fs {0}'.format(device)
    if args:
        cmd = cmd + ' -' + args
    ret = {'attributes': {}, 'blocks': {}}
    out = __salt__['cmd.run'](cmd, python_shell=False).splitlines()
    mode = 'opts'
    group = None
    for line in out:
        if not line:
            continue
        if line.startswith('dumpe2fs'):
            continue
        if mode == 'opts':
            line = line.replace('\t', ' ')
            comps = line.split(': ')
            if line.startswith('Filesystem features'):
                ret['attributes'][comps[0]] = comps[1].split()
            elif line.startswith('Group') and not line.startswith('Group descriptor size'):
                mode = 'blocks'
            else:
                if len(comps) < 2:
                    continue
                ret['attributes'][comps[0]] = comps[1].strip()

        if mode == 'blocks':
            if line.startswith('Group'):
                line = line.replace(':', '')
                line = line.replace('(', '')
                line = line.replace(')', '')
                line = line.replace('[', '')
                line = line.replace(']', '')
                comps = line.split()
                blkgrp = comps[1]
                group = 'Group {0}'.format(blkgrp)
                ret['blocks'][group] = {}
                ret['blocks'][group]['group'] = blkgrp
                ret['blocks'][group]['range'] = comps[3]
                # TODO: comps[4:], which may look one one of the following:
                #     ITABLE_ZEROED
                #     INODE_UNINIT, ITABLE_ZEROED
                # Does anyone know what to call these?
                ret['blocks'][group]['extra'] = []
            elif 'Free blocks:' in line:
                comps = line.split(': ')
                free_blocks = comps[1].split(', ')
                ret['blocks'][group]['free blocks'] = free_blocks
            elif 'Free inodes:' in line:
                comps = line.split(': ')
                inodes = comps[1].split(', ')
                ret['blocks'][group]['free inodes'] = inodes
            else:
                line = line.strip()
                ret['blocks'][group]['extra'].append(line)
    return ret
948581
def _prepare_output(partitions, unavailable_brokers, verbose):
    """"""
    partitions_count = len(partitions)
    out = {}
    out['raw'] = {
        'replica_unavailability_count': partitions_count,
    }

    if partitions_count == 0:
        out['message'] = 'All replicas available for communication.'
    else:
        out['message'] = "{replica_unavailability} replicas unavailable for communication. " \
            "Unavailable Brokers: {unavailable_brokers}".format(
            replica_unavailability=partitions_count,
            unavailable_brokers=', '.join([str(e) for e in unavailable_brokers]),
        )
        if verbose:
            lines = (
                '{}:{}'.format(topic, partition)
                for (topic, partition) in partitions
            )
            out['verbose'] = "Partitions:\n" + "\n".join(lines)

    if verbose:
        out['raw']['partitions'] = [
            {'topic': topic, 'partition': partition}
            for (topic, partition) in partitions
        ]

    return out
14678
def _prepare_disks(self, disks_name):
        """"""
        fstab = '/etc/fstab'
        for disk in tqdm(disks_name.split(',')):
            sudo('umount /dev/{0}'.format(disk), warn_only=True)
            if sudo('mkfs.xfs -f /dev/{0}'.format(disk), warn_only=True).failed:
                sudo('apt-get update')
                sudo('apt-get -y install xfsprogs')
                sudo('mkfs.xfs -f /dev/{0}'.format(disk))
            sudo('mkdir -p /srv/node/{0}'.format(disk))
            files.append(
                fstab, '/dev/{0} /srv/node/{1} xfs noatime,nodiratime,nobarrier,logbufs=8 0 2'.format(disk, disk), use_sudo=True)
            sudo('mount /srv/node/{0}'.format(disk))
938553
def _save_payload(self, files, directories, links):
        '''
        
        '''

        idx = 0
        for p_type, p_list in (('f', files), ('d', directories), ('l', links,),):
            for p_obj in p_list:
                stats = os.stat(p_obj)

                payload = PayloadFile()
                payload.id = idx
                payload.path = p_obj
                payload.p_type = p_type
                payload.mode = stats.st_mode
                payload.uid = stats.st_uid
                payload.gid = stats.st_gid
                payload.p_size = stats.st_size
                payload.atime = stats.st_atime
                payload.mtime = stats.st_mtime
                payload.ctime = stats.st_ctime

                idx += 1
                self.db.store(payload)
591542
def dump_bulk(cls, parent=None, keep_ids=True):
        """"""

        cls = get_result_class(cls)

        # Because of fix_tree, this method assumes that the depth
        # and numchild properties in the nodes can be incorrect,
        # so no helper methods are used
        qset = cls._get_serializable_model().objects.all()
        if parent:
            qset = qset.filter(path__startswith=parent.path)
        ret, lnk = [], {}
        for pyobj in serializers.serialize('python', qset):
            # django's serializer stores the attributes in 'fields'
            fields = pyobj['fields']
            path = fields['path']
            depth = int(len(path) / cls.steplen)
            # this will be useless in load_bulk
            del fields['depth']
            del fields['path']
            del fields['numchild']
            if 'id' in fields:
                # this happens immediately after a load_bulk
                del fields['id']

            newobj = {'data': fields}
            if keep_ids:
                newobj['id'] = pyobj['pk']

            if (not parent and depth == 1) or\
               (parent and len(path) == len(parent.path)):
                ret.append(newobj)
            else:
                parentpath = cls._get_basepath(path, depth - 1)
                parentobj = lnk[parentpath]
                if 'children' not in parentobj:
                    parentobj['children'] = []
                parentobj['children'].append(newobj)
            lnk[path] = newobj
        return ret
146493
def dumps(self):
        """"""
        self.data.update({
            'bucket': str(self.obj.bucket_id),
            'checksum': self.obj.file.checksum,
            'key': self.obj.key,  # IMPORTANT it must stay here!
            'size': self.obj.file.size,
            'version_id': str(self.obj.version_id),
        })
        return self.data
504302
def dump(self):
        
        results = []
        for data in self.data():
            results.append(data)
        return results
134668
def safe_dump(data, stream=None, **kwds):
    
    return dump_all([data], stream, Dumper=SafeDumper, **kwds)
1007107
def dump(self, directory):
    """"""
    payload = '{}:{}'.format(self.fingerprint, self.serialized_bytes_length)
    safe_file_dump(self._path(directory), payload=payload)
447625
def export(self):
        
        with LogTask('Exporting disk {} to {}'.format(self.name, self.dst)):
            with utils.RollbackContext() as rollback:
                rollback.prependDefer(
                    shutil.rmtree, self.dst, ignore_errors=True
                )
                self.copy()
                if not self.disk['format'] == 'iso':
                    self.sparse()
                self.calc_sha('sha1')
                self.update_lago_metadata()
                self.write_lago_metadata()
                self.compress()
                rollback.clear()
531418
def db_export(schema, uuid, object_filter, export_format, filename, pretty, all_schemata, omit):
    """"""

    internal_backup(schema, uuid, object_filter, export_format, filename, pretty, all_schemata, omit)
986622
def dump(self):
        """"""
        self.print("Dumping data to {}".format(self.dump_filename))
        pickle.dump({
            'data': self.counts,
            'livetime': self.get_livetime()
        }, open(self.dump_filename, "wb"))
809259
def dump_t_coords(dataset_dir, data_dir, dataset, root=None, compress=True):
    """"""
    if root is None:
        root = {}
    tcoords = dataset.GetPointData().GetTCoords()
    if tcoords:
        dumped_array = dump_data_array(dataset_dir, data_dir, tcoords, {}, compress)
        root['pointData']['activeTCoords'] = len(root['pointData']['arrays'])
        root['pointData']['arrays'].append({'data': dumped_array})
926034
def backup(self):
        """"""
        count = 0
        name = "{}.bkp".format(self.filename)
        backup = os.path.join(self.cwd, name)
        while os.path.exists(backup):
            count += 1
            name = "{}.bkp{}".format(self.filename, count)
            backup = os.path.join(self.cwd, name)
        self.hey("Moving existing {} to {}".format(self.filename, name))
        os.rename(os.path.join(self.cwd, self.filename), backup)