Reduce a counts cube to a counts map
355410
def sum_over_energy(self):
        
        # We sum over axis 0 in the array, and drop the energy binning in the
        # hpx object
        return HpxMap(np.sum(self.counts, axis=0), self.hpx.copy_and_drop_energy())
118979
def expanded_counts_map(self):
        
        if self.hpx._ipix is None:
            return self.counts

        output = np.zeros(
            (self.counts.shape[0], self.hpx._maxpix), self.counts.dtype)
        for i in range(self.counts.shape[0]):
            output[i][self.hpx._ipix] = self.counts[i]
        return output
144171
def col_frequencies(col, weights=None, gap_chars='-.'):
    """"""
    counts = col_counts(col, weights, gap_chars)
    # Reduce to frequencies
    scale = 1.0 / sum(counts.values())
    return dict((aa, cnt * scale) for aa, cnt in counts.iteritems())
854650
def build_maps(self, losses, clp, stats=()):
        
        shp = losses.shape[:2] + (len(clp), len(losses.dtype))  # (A, R, C, LI)
        array = numpy.zeros(shp, F32)
        for lti, lt in enumerate(losses.dtype.names):
            for a, losses_ in enumerate(losses[lt]):
                for r, ls in enumerate(losses_):
                    for c, poe in enumerate(clp):
                        clratio = conditional_loss_ratio(ls, self.poes, poe)
                        array[a, r, c, lti] = clratio
        return self.pair(array, stats)
590787
def _counts(self):
        

        rolling_dim = utils.get_temp_dimname(self.obj.dims, '_rolling_dim')
        # We use False as the fill_value instead of np.nan, since boolean
        # array is faster to be reduced than object array.
        # The use of skipna==False is also faster since it does not need to
        # copy the strided array.
        counts = (self.obj.notnull()
                  .rolling(center=self.center, **{self.dim: self.window})
                  .construct(rolling_dim, fill_value=False)
                  .sum(dim=rolling_dim, skipna=False))
        return counts
832946
def _data(self, cube, weighted, prune):
        """"""
        result = []
        for slice_ in cube.slices:
            if cube.has_mr:
                return self._mr_index(cube, weighted, prune)
            num = slice_.margin(axis=0, weighted=weighted, prune=prune)
            den = slice_.margin(weighted=weighted, prune=prune)
            margin = num / den
            proportions = slice_.proportions(axis=1, weighted=weighted, prune=prune)
            result.append(proportions / margin)

        if len(result) == 1 and cube.ndim < 3:
            result = result[0]
        else:
            if prune:
                mask = np.array([slice_.mask for slice_ in result])
                result = np.ma.masked_array(result, mask)
            else:
                result = np.array(result)

        return result
1282018
def _reduce_input(self, inputs, reducer, final=NotImplemented):
        
        for key, values in groupby(inputs, key=lambda x: self.internal_serialize(x[0])):
            for output in reducer(self.deserialize(key), (v[1] for v in values)):
                yield output
        if final != NotImplemented:
            for output in final():
                yield output
        self._flush_batch_incr_counter()
789385
def visualize_qual_stats_dict_single(D, dest, title):
    

    # first find out how many cycles were there. it is going to be about 101 for
    # hiseq runs, and 251 in miseq runs, but these values may change from run to
    # run. although all lanes are expected to have the identical number of cycles
    # the following code makes sure that the number_of_cycles variable holds the
    # longest one if there is a variation between the number of cycles between
    # lanes
    number_of_cycles = 0
    for pair in ['1', '2']:
        if pair not in D:
            continue
        for tile in D[pair]:
            if len(D[pair][tile]['mean']) > number_of_cycles:
                number_of_cycles = len(D[pair][tile]['mean'])


    fig = plt.figure(figsize = (12, 8))

    plt.rcParams.update({'axes.linewidth' : 0.9})
    plt.rc('grid', color='0.50', linestyle='-', linewidth=0.1)

    all_tiles = {'1': {'mean': [0] * number_of_cycles, 'count': [0] * number_of_cycles},
                 '2': {'mean': [0] * number_of_cycles, 'count': [0] * number_of_cycles}
                }

    for i in range(0, number_of_cycles):
        means_p1 = []
        counts_p1 = []
        means_p2 = []
        counts_p2 = []
        for tile_id in D['1']:
            tile = D['1'][tile_id]
            means_p1.append(tile['mean'][i])
            counts_p1.append(tile['count'][i])
            if '2' in D and D['2']:
                tile = D['2'][tile_id]
                means_p2.append(tile['mean'][i])
                counts_p2.append(tile['count'][i])

        all_tiles['1']['mean'][i] = numpy.mean(means_p1)
        all_tiles['1']['count'][i] = sum(counts_p1)

        if '2' in D and D['2']:
            all_tiles['2']['mean'][i] = numpy.mean(means_p2)
            all_tiles['2']['count'][i] = sum(counts_p2)

    colors = cm.get_cmap('RdYlGn', lut=256)

    plt.grid(True)

    plt.subplots_adjust(left=0.02, bottom = 0.03, top = 0.95, right = 0.98)

    plt.xticks(list(range(number_of_cycles / 10, number_of_cycles, number_of_cycles / 10)), rotation=90, size='xx-small')
    plt.ylim(ymin = 0, ymax = 42)
    plt.xlim(xmin = 0, xmax = number_of_cycles - 1)
    plt.yticks(list(range(5, 41, 5)), size='xx-small')

    plt.fill_between(list(range(0, number_of_cycles)), [42 for _ in range(0, number_of_cycles)], y2 = 0, color = colors(0), alpha = 0.2)
    plt.plot(all_tiles['1']['mean'], color = 'orange', lw = 6)

    read_number_percent_dropdown = [42 * (x / all_tiles['1']['count'][0]) for x in all_tiles['1']['count']]
    if not len(set(read_number_percent_dropdown)) <= 1:
        plt.fill_between(list(range(0, number_of_cycles)), read_number_percent_dropdown, y2 = 0, color = 'black', alpha = 0.08)

        plt.text(5, 2.5, '%s' % big_number_pretty_print(all_tiles['1']['count'][0]), alpha=0.5)
    else:
        plt.text(5, 2.5, '%s' % big_number_pretty_print(all_tiles['1']['count'][0]), alpha=0.5)

    if '2' in all_tiles and all_tiles['2']:
        plt.plot(all_tiles['2']['mean'], color = 'purple', lw = 6)

    plt.figtext(0.5, 0.97, '%s' % (title), weight = 'black', size = 'xx-large', ha = 'center')

    try:
        plt.savefig(dest + '.tiff')
    except:
        plt.savefig(dest + '.png')

    return (all_tiles['1']['mean'], all_tiles['2']['mean'])
789995
def _counts_to_amber(t_vals, n_vals):
    
    t_depth = int(t_vals["REF_COUNT"]) + int(t_vals["ALT_COUNT"])
    n_depth = int(n_vals["REF_COUNT"]) + int(n_vals["ALT_COUNT"])
    if n_depth > 0 and t_depth > 0:
        t_baf = float(t_vals["ALT_COUNT"]) / float(t_depth)
        n_baf = float(n_vals["ALT_COUNT"]) / float(n_depth)
        return [t_vals["CONTIG"], t_vals["POSITION"], t_baf, _normalize_baf(t_baf), t_depth,
                n_baf, _normalize_baf(n_baf), n_depth]
46476
def count(self):
        
        return self.mapPartitions(lambda i: [sum(1 for _ in i)]).reduce(operator.add)
362480
def reduce(self, dimensions=None, function=None, **reduce_map):
        
        function, dims = self._reduce_map(dimensions, function, reduce_map)
        if len(dims) == self.ndims:
            if isinstance(function, np.ufunc):
                return function.reduce(self.data, axis=None)
            else:
                return function(self.data)
        else:
            dimension = dims[0]
            other_dimension = [d for d in self.kdims if d.name != dimension]
            oidx = self.get_dimension_index(other_dimension[0])
            x_vals = self.dimension_values(other_dimension[0].name, False)
            reduced = function(self._zdata, axis=oidx)
            if oidx and hasattr(self, 'bounds'):
                reduced = reduced[::-1]
            data = zip(x_vals, reduced)
            params = dict(dict(self.get_param_values(onlychanged=True)),
                          kdims=other_dimension, vdims=self.vdims)
            params.pop('bounds', None)
            params.pop('extents', None)
            return Table(data, **params)
872577
def tables_with_counts(self):
        """"""
        table_to_count = lambda t: self.count_rows(t)
        return zip(self.tables, map(table_to_count, self.tables))
230773
def NdarrayToEntry(self, x):
    """"""
    row_counts = []
    for row in x:
      try:
        rc = np.count_nonzero(~np.isnan(row))
        if rc != 0:
          row_counts.append(rc)
      except TypeError:
        try:
          row_counts.append(row.size)
        except AttributeError:
          row_counts.append(1)

    data_type = self.DtypeToType(x.dtype)
    converter = self.DtypeToNumberConverter(x.dtype)
    flattened = x.ravel()
    orig_size = len(flattened)

    # Remove all None and nan values and count how many were removed.
    flattened = flattened[flattened != np.array(None)]
    if converter:
      flattened = converter(flattened)
    if data_type == self.fs_proto.STRING:
      flattened_temp = []
      for x in flattened:
        try:
          if str(x) != 'nan':
            flattened_temp.append(x)
        except UnicodeEncodeError:
          if x.encode('utf-8') != 'nan':
            flattened_temp.append(x)
      flattened = flattened_temp
    else:
      flattened = flattened[~np.isnan(flattened)].tolist()
    missing = orig_size - len(flattened)
    return {
        'vals': flattened,
        'counts': row_counts,
        'missing': missing,
        'type': data_type
    }
1074609
def count(self):
        
        return (
            self
            .mapPartitions(lambda p: [sum(1 for _ in p)])
            .reduce(operator.add)
        )
30986
def number(items):
    """"""
    n = len(items)
    if n == 0:
        return items
    places = str(int(math.log10(n) // 1 + 1))
    format = '[{0[0]:' + str(int(places)) + 'd}] {0[1]}'
    return map(
        lambda x: format.format(x),
        enumerate(items)
    )
559359
def _perform_reduce(self, map_results, factory, group=False,
                        group_level=None):
        '''
        
        '''

        def get_group_key(key, group, group_level):
            if group:
                return key
            return key[0:group_level]

        if not group and group_level is None:
            keys = map(operator.itemgetter(0), map_results)
            values = map(operator.itemgetter(1), map_results)
            return self._reduce_values(factory, None, keys, values)
        else:
            groups = dict()
            for row in map_results:
                key, value = row[:2]
                group_key = get_group_key(key, group, group_level)
                if group_key not in groups:
                    groups[group_key] = list()
                groups[group_key].append((key, value))
            resp = list()
            for group_key, results in groups.iteritems():
                keys = map(operator.itemgetter(0), results)
                values = map(operator.itemgetter(1), results)
                resp.extend(
                    self._reduce_values(factory, group_key, keys, values))
            return resp
18855
function(T, SA, C, B, n, k) {
        var b, i, j;
        var c0, c1;
        /* compute SAl */
        if (C === B) { getCounts(T, C, n, k); }
        getBuckets(C, B, k, false); /* find starts of buckets */
        j = n - 1;
        b = B[c1 = T[j]];
        j--;
        SA[b++] = (T[j] < c1) ? ~j : j;
        for (i = 0; i < n; i++) {
            if ((j = SA[i]) > 0) {
                ASSERT(T[j] >= T[j+1]);
                if ((c0 = T[j]) !== c1) { B[c1] = b; b = B[c1 = c0]; }
                ASSERT(i < b);
                j--;
                SA[b++] = (T[j] < c1) ? ~j : j;
                SA[i] = 0;
            } else if (j < 0) {
                SA[i] = ~j;
            }
        }
        /* compute SAs */
        if (C === B) { getCounts(T, C, n, k); }
        getBuckets(C, B, k, 1); /* find ends of buckets */
        for (i = n-1, b = B[c1 = 0]; i >= 0; i--) {
            if ((j = SA[i]) > 0) {
                ASSERT(T[j] <= T[j+1]);
                if ((c0 = T[j]) !== c1) { B[c1] = b; b = B[c1 = c0]; }
                ASSERT(b <= i);
                j--;
                SA[--b] = (T[j] > c1) ? ~(j+1) : j;
                SA[i] = 0;
            }
        }
    }