Retrieve number of regions that can be processed in parallel from current samples.
355244
def get_max_counts(samples):
    
    counts = []
    for data in (x[0] for x in samples):
        count = tz.get_in(["config", "algorithm", "callable_count"], data, 1)
        vcs = tz.get_in(["config", "algorithm", "variantcaller"], data, [])
        if isinstance(vcs, six.string_types):
            vcs = [vcs]
        if vcs:
            count *= len(vcs)
        counts.append(count)
    return max(counts)
854444
public int samplesAvailableToEncode() {
    int available = 0;
    //sum all in blockQueue
    int channels = streamConfig.getChannelCount();
    for(BlockEncodeRequest ber : preparedRequests) {
      int[] block = ber.samples;
      available += block.length/channels;
    }
    if (unfilledRequest != null)
      available += unfilledRequest.count;
    return available;
  }
1338418
def load_results(result_files, options, run_set_id=None, columns=None,
                 columns_relevant_for_diff=set()):
    """"""
    return parallel.map(
        load_result,
        result_files,
        itertools.repeat(options),
        itertools.repeat(run_set_id),
        itertools.repeat(columns),
        itertools.repeat(columns_relevant_for_diff))
82371
def _process_non_parallel(self, X: Union[pd.DataFrame, np.ndarray], n_refs: int, cluster_array: np.ndarray):
        
        for gap_value, n_clusters in [self._calculate_gap(X, n_refs, n_clusters)
                                      for n_clusters in cluster_array]:
            yield (gap_value, n_clusters)
768893
def process_buffer(buffer, n_channels):
    
    samples = np.concatenate(buffer)

    if n_channels > 1:
        samples = samples.reshape((-1, n_channels)).T
        samples = librosa.to_mono(samples)

    return samples
675861
def sampled_data(self):
        """"""
        if self.args.samples_file:
            if self.args.invert_samples:
                chosen_samples = set(self.hash_to_ind) - self.samples
            else:
                chosen_samples = self.samples
            selected_indices = [self.hash_to_ind[h] for h in chosen_samples]
            return self.train[0][selected_indices], self.train[1][selected_indices]
        else:
            return self.train[0], self.train[1]
341763
def _get_num_parallel_threads(max_threads, num_cores, mem_available_mb):
    '''
    
    '''
    return min(max_threads, num_cores, max(int(mem_available_mb/1200), 1))
180542
def packet_get_samples_per_frame(data, fs):
    """"""

    data_pointer = ctypes.c_char_p(data)

    result = _packet_get_nb_frames(data_pointer, ctypes.c_int(fs))
    if result < 0:
        raise OpusError(result)

    return result
683940
protected int readSamples ()
        throws IOException
    {
        int samples;
        while ((samples = _dsp.synthesis_pcmout(_pcm, _offsets)) <= 0) {
            if (samples == 0 && !readPacket()) {
                return 0;
            }
            if (_block.synthesis(_packet) == 0) {
                _dsp.synthesis_blockin(_block);
            }
        }
        return samples;
    }
1134830
private int sample(Queue<OutboundFrame> q) {
        packets.addAll(q);

        if (packets.size() < threshold) {
            return -1;
        }

        int sampleCount = min(samples, packets.size());
        int actualSampleCount = 0;
        for (int k = 0; k < sampleCount; k++) {
            OutboundFrame packet = packets.get(random.nextInt(packets.size()));
            String key = toKey(packet);

            if (key != null) {
                actualSampleCount++;
                occurrenceMap.add(key, 1);
            }
        }

        return actualSampleCount;
    }
752941
def get_measured_regions(self):
        
        mergeon = ['project_id','project_name',
                'sample_id','sample_name',
                'frame_id','frame_name',
                ]
        temp = self.loc[:,mergeon+['regions']].\
            set_index(mergeon)['regions'].apply(json.dumps).\
            reset_index().drop_duplicates()
        temp['regions'] = temp['regions'].apply(json.loads)
        rows = []
        for i,r in temp.iterrows():
            for label in r['regions']:
                a = list(r.index)
                b = list(r.values)
                a = a+['region_label','region_area_pixels']
                b = b+[label,r['regions'][label]]
                rows.append(dict(zip(a,b)))
        rows = pd.DataFrame(rows).drop(columns='regions').\
            drop_duplicates()[mergeon+['region_label','region_area_pixels']]
        #rows = rows.loc[rows['region_area_pixels']>0].copy()
        return rows
1305904
private Iterator<MultiTableScanResult> scanMultiTableRows(
            final TableSet tables, final DeltaPlacement placement, final ByteBufferRange rowRange,
            final LimitCounter limit, final boolean includeDroppedTables, final boolean includeMirrorTables,
            final int largeRowThreshold, final ReadConsistency consistency, @Nullable final Instant cutoffTime) {

        // Avoiding pinning multiple decoded rows into memory at once.
        return limit.limit(new AbstractIterator<MultiTableScanResult>() {
            private PeekingIterator<Row<ByteBuffer, DeltaKey>> _iter = Iterators.peekingIterator(
                    rowScan(placement, rowRange, _maxColumnsRange, LimitCounter.max(), consistency));

            private long _lastTableUuid = -1;
            private AstyanaxTable _table = null;
            private boolean _droppedTable;
            private boolean _primaryTable;

            @Override
            protected MultiTableScanResult computeNext() {
                while (_iter.hasNext()) {
                    Row<ByteBuffer, DeltaKey> row = _iter.next();
                    ColumnList<DeltaKey> rowColumns = row.getColumns();

                    // Convert the results into a Record object, lazily fetching the rest of the columns as necessary.
                    ByteBuffer rowKey = row.getRawKey();

                    long tableUuid = AstyanaxStorage.getTableUuid(rowKey);
                    if (_lastTableUuid != tableUuid) {
                        _lastTableUuid = tableUuid;
                        try {
                            _table = (AstyanaxTable) tables.getByUuid(tableUuid);
                        } catch (UnknownTableException e) {
                            _table = AstyanaxTable.createUnknown(tableUuid, placement, e.getTable());
                        } catch (DroppedTableException e) {
                            _table = AstyanaxTable.createUnknown(tableUuid, placement, e.getPriorTable());
                        }
                        _droppedTable = _table.isUnknownTable();
                        _primaryTable = _table.getReadStorage().hasUUID(tableUuid);
                    }

                    // Skip dropped and mirror tables if configured
                    if ((!includeDroppedTables && _droppedTable) || (!includeMirrorTables && !_primaryTable)) {
                        _iter = skipToNextTable(tableUuid);
                        continue;
                    }

                    int shardId = AstyanaxStorage.getShardId(rowKey);
                    String key = AstyanaxStorage.getContentKey(rowKey);
                    Record record = newRecord(new Key(_table, key), rowKey, rowColumns, largeRowThreshold, consistency, cutoffTime);
                    return new MultiTableScanResult(rowKey, shardId, tableUuid, _droppedTable, record);
                }

                return endOfData();
            }

            private PeekingIterator<Row<ByteBuffer, DeltaKey>> skipToNextTable(long tableUuid) {
                // Iterate over the next 50 rows first to check for a table switch.  This avoids starting a new range
                // query if the number of rows in the undesired table is small.
                int skipLimit = 50;
                Row<ByteBuffer, DeltaKey> row = null;

                while (skipLimit != 0 && _iter.hasNext()) {
                    row = _iter.peek();
                    long nextTableUuid = AstyanaxStorage.getTableUuid(row.getRawKey());
                    if (nextTableUuid != tableUuid) {
                        // This is the first row of a new table
                        return _iter;
                    } else {
                        _iter.next();
                        skipLimit -= 1;
                    }
                }

                if (_iter.hasNext()) {
                    // Skip the table entirely by starting a new query on the next possible table
                    assert row != null;
                    int shardId = AstyanaxStorage.getShardId(row.getRawKey());
                    ByteBuffer nextPossibleTableStart = AstyanaxStorage.getRowKeyRaw(shardId, tableUuid + 1, "");
                    ByteBuffer end = rowRange.getEnd();

                    if (AstyanaxStorage.compareKeys(nextPossibleTableStart, end) < 0) {
                        // We haven't reached the last end boundary of the original range scan
                        ByteBufferRange updatedRange = new ByteBufferRangeImpl(nextPossibleTableStart, end, -1, false);
                        return Iterators.peekingIterator(
                                rowScan(placement, updatedRange, _maxColumnsRange, LimitCounter.max(), consistency));
                    }
                }

                return Iterators.peekingIterator(Iterators.<Row<ByteBuffer, DeltaKey>>emptyIterator());
            }
        });
    }
893205
def _analysis_block_stats(regions, samples):
    
    prev = None
    between_sizes = []
    region_sizes = []
    for region in regions:
        if prev and prev.chrom == region.chrom:
            between_sizes.append(region.start - prev.end)
        region_sizes.append(region.end - region.start)
        prev = region
    def descriptive_stats(xs):
        if len(xs) < 2:
            return xs
        parts = ["min: %s" % min(xs),
                 "5%%: %s" % numpy.percentile(xs, 5),
                 "25%%: %s" % numpy.percentile(xs, 25),
                 "median: %s" % numpy.percentile(xs, 50),
                 "75%%: %s" % numpy.percentile(xs, 75),
                 "95%%: %s" % numpy.percentile(xs, 95),
                 "99%%: %s" % numpy.percentile(xs, 99),
                 "max: %s" % max(xs)]
        return "\n".join(["  " + x for x in parts])
    logger.info("Identified %s parallel analysis blocks\n" % len(region_sizes) +
                "Block sizes:\n%s\n" % descriptive_stats(region_sizes) +
                "Between block sizes:\n%s\n" % descriptive_stats(between_sizes))
    if len(region_sizes) == 0:
        raise ValueError("No callable regions found in: %s" %
                         (", ".join([dd.get_sample_name(x) for x in samples])))
983805
def _sampler_n_samples(self, n_samples):
        
        sampler_indices = self.rng_.choice(range(len(self.samplers)),
                                           size=n_samples,
                                           replace=True,
                                           p=self.weights)
        return [
            (self.samplers[idx], freq)
            for idx, freq in itemfreq(sampler_indices)
        ]
943987
def generate_parallel(samples, run_parallel):
    
    to_analyze, extras = _split_samples_by_qc(samples)
    qced = run_parallel("pipeline_summary", to_analyze)
    samples = _combine_qc_samples(qced) + extras
    qsign_info = run_parallel("qsignature_summary", [samples])
    metadata_file = _merge_metadata([samples])
    summary_file = write_project_summary(samples, qsign_info)
    out = []
    for data in samples:
        if "summary" not in data[0]:
            data[0]["summary"] = {}
        data[0]["summary"]["project"] = summary_file
        data[0]["summary"]["metadata"] = metadata_file
        if qsign_info:
            data[0]["summary"]["mixup_check"] = qsign_info[0]["out_dir"]
        out.append(data)
    out = _add_researcher_summary(out, summary_file)
    # MultiQC must be run after all file outputs are set:
    return [[utils.to_single_data(d)] for d in run_parallel("multiqc_summary", [out])]
1232888
def sample(self, n):
    

    row_total_count = 0
    row_counts = []
    for file in self.files:
      with _util.open_local_or_gcs(file, 'r') as f:
        num_lines = sum(1 for line in f)
        row_total_count += num_lines
        row_counts.append(num_lines)

    names = None
    dtype = None
    if self._schema:
      _MAPPINGS = {
        'FLOAT': np.float64,
        'INTEGER': np.int64,
        'TIMESTAMP': np.datetime64,
        'BOOLEAN': np.bool,
      }
      names = [x['name'] for x in self._schema]
      dtype = {x['name']: _MAPPINGS.get(x['type'], object) for x in self._schema}

    skip_count = row_total_count - n
    # Get all skipped indexes. These will be distributed into each file.
    # Note that random.sample will raise Exception if skip_count is greater than rows count.
    skip_all = sorted(random.sample(range(0, row_total_count), skip_count))
    dfs = []
    for file, row_count in zip(self.files, row_counts):
      skip = [x for x in skip_all if x < row_count]
      skip_all = [x - row_count for x in skip_all if x >= row_count]
      with _util.open_local_or_gcs(file, 'r') as f:
        dfs.append(pd.read_csv(f, skiprows=skip, names=names, dtype=dtype, header=None))
    return pd.concat(dfs, axis=0, ignore_index=True)
682110
def get_variants_in_region(self, chrom, start, end):
        """"""
        region = self.get_vcf()(
            "{}:{}-{}".format(chrom, start, end)
        )
        for v in region:
            for coded_allele, g in self._make_genotypes(v.ALT, v.genotypes):
                variant = Variant(
                    v.ID, v.CHROM, v.POS, [v.REF, coded_allele]
                )
                yield Genotypes(variant, g, v.REF, coded_allele,
                                multiallelic=len(v.ALT) > 1)