Run searchlight function on block data in parallel.

    `extra_params` contains:

    - Searchlight function.
    - `Shape` mask.
    - Minimum active voxels proportion required to run the searchlight
      function.
12018
def _singlenode_searchlight(l, msk, mysl_rad, bcast_var, extra_params):
    

    voxel_fn = extra_params[0]
    shape_mask = extra_params[1]
    min_active_voxels_proportion = extra_params[2]
    outmat = np.empty(msk.shape, dtype=np.object)[mysl_rad:-mysl_rad,
                                                  mysl_rad:-mysl_rad,
                                                  mysl_rad:-mysl_rad]
    for i in range(0, outmat.shape[0]):
        for j in range(0, outmat.shape[1]):
            for k in range(0, outmat.shape[2]):
                if msk[i+mysl_rad, j+mysl_rad, k+mysl_rad]:
                    searchlight_slice = np.s_[
                        i:i+2*mysl_rad+1,
                        j:j+2*mysl_rad+1,
                        k:k+2*mysl_rad+1]
                    voxel_fn_mask = msk[searchlight_slice] * shape_mask
                    if (min_active_voxels_proportion == 0
                        or np.count_nonzero(voxel_fn_mask) / voxel_fn_mask.size
                            > min_active_voxels_proportion):
                        outmat[i, j, k] = voxel_fn(
                            [ll[searchlight_slice] for ll in l],
                            msk[searchlight_slice] * shape_mask,
                            mysl_rad,
                            bcast_var)
    return outmat
646027
def run(items, background=None):
    
    if not background: background = []
    background_bams = []
    paired = vcfutils.get_paired_bams([x["align_bam"] for x in items], items)
    if paired:
        inputs = [paired.tumor_data]
        if paired.normal_bam:
            background = [paired.normal_data]
            background_bams = [paired.normal_bam]
    else:
        assert not background
        inputs, background = shared.find_case_control(items)
        background_bams = [x["align_bam"] for x in background]
    orig_vcf = _run_wham(inputs, background_bams)
    out = []
    for data in inputs:
        if "sv" not in data:
            data["sv"] = []
        final_vcf = shared.finalize_sv(orig_vcf, data, items)
        data["sv"].append({"variantcaller": "wham", "vrn_file": final_vcf})
        out.append(data)
    return out
812745
def _make_bright_pixel_mask(intensity_mean, mask_factor=5.0):
        
        mask = np.zeros((intensity_mean.data.shape), bool)
        nebins = len(intensity_mean.data)
        sum_intensity = intensity_mean.data.sum(0)
        mean_intensity = sum_intensity.mean()
        for i in range(nebins):
            mask[i, 0:] = sum_intensity > (mask_factor * mean_intensity)
        return HpxMap(mask, intensity_mean.hpx)
358644
def _filter_extras(dm):
        
        for extra in list(filter(None, dm)):
            new_extra = extra
            reqs = dm.pop(extra)
            new_extra, _, marker = extra.partition(':')
            fails_marker = marker and (
                invalid_marker(marker)
                or not evaluate_marker(marker)
            )
            if fails_marker:
                reqs = []
            new_extra = safe_extra(new_extra) or None

            dm.setdefault(new_extra, []).extend(reqs)
        return dm
29853
def run(samples, run_parallel):
    
    to_process = []
    extras = []
    for data in (xs[0] for xs in samples):
        hlacaller = tz.get_in(["config", "algorithm", "hlacaller"], data)
        if hlacaller:
            to_process.append(data)
        else:
            extras.append([data])
    processed = run_parallel("call_hla", ([x] for x in to_process))
    return extras + processed
650503
def _extrapolate_rows(self, data, row_indices, first_index, last_index):
        

        pos = row_indices[:2]
        first_row = _linear_extrapolate(pos,
                                        (data[0, :], data[1, :]),
                                        first_index)
        pos = row_indices[-2:]
        last_row = _linear_extrapolate(pos,
                                       (data[-2, :], data[-1, :]),
                                       last_index)
        return np.vstack((np.expand_dims(first_row, 0),
                          data,
                          np.expand_dims(last_row, 0)))
1042886
private static void thresholdBlock(byte[] luminances,
                                     int xoffset,
                                     int yoffset,
                                     int threshold,
                                     int stride,
                                     BitMatrix matrix) {
    for (int y = 0, offset = yoffset * stride + xoffset; y < BLOCK_SIZE; y++, offset += stride) {
      for (int x = 0; x < BLOCK_SIZE; x++) {
        // Comparison needs to be <= so that black == 0 pixels are black even if the threshold is 0.
        if ((luminances[offset + x] & 0xFF) <= threshold) {
          matrix.set(xoffset + x, yoffset + y);
        }
      }
    }
  }
611290
def run_parallel_map_providers_query(data, queue=None):
    '''
    
    '''
    salt.utils.crypt.reinit_crypto()

    cloud = Cloud(data['opts'])
    try:
        with salt.utils.context.func_globals_inject(
            cloud.clouds[data['fun']],
            __active_provider_name__=':'.join([
                data['alias'],
                data['driver']
            ])
        ):
            return (
                data['alias'],
                data['driver'],
                salt.utils.data.simple_types_filter(
                    cloud.clouds[data['fun']]()
                )
            )
    except Exception as err:
        log.debug(
            'Failed to execute \'%s()\' while querying for running nodes: %s',
            data['fun'], err, exc_info_on_loglevel=logging.DEBUG
        )
        # Failed to communicate with the provider, don't list any nodes
        return data['alias'], data['driver'], ()
841542
private static void setDpiValue(final Multimap<String, String> extraParams, final int dpi) {
        String searchKey = "FORMAT_OPTIONS";
        for (String key: extraParams.keys()) {
            if (key.equalsIgnoreCase(searchKey)) {
                Collection<String> values = extraParams.removeAll(key);
                List<String> newValues = new ArrayList<>();
                for (String value: values) {
                    if (!StringUtils.isEmpty(value)) {
                        value += ";dpi:" + Integer.toString(dpi);
                        newValues.add(value);
                    }
                }
                extraParams.putAll(key, newValues);
                return;
            }
        }
    }
483928
def _extrapolate_cols(self, data, first=True, last=True):
        

        if first:
            pos = self.col_indices[:2]
            first_column = _linear_extrapolate(pos,
                                               (data[:, 0], data[:, 1]),
                                               self.hcol_indices[0])
        if last:
            pos = self.col_indices[-2:]
            last_column = _linear_extrapolate(pos,
                                              (data[:, -2], data[:, -1]),
                                              self.hcol_indices[-1])

        if first and last:
            return np.hstack((np.expand_dims(first_column, 1),
                              data,
                              np.expand_dims(last_column, 1)))
        elif first:
            return np.hstack((np.expand_dims(first_column, 1),
                              data))
        elif last:
            return np.hstack((data,
                              np.expand_dims(last_column, 1)))
        else:
            return data
819176
def extra(self, **params):
        
        q = self._clone()
        for key, value in params.items():
            q._extra[key].append(value)
        return q
1002605
def call_parallel(self, cdata, low):
        '''
        
        '''
        # There are a number of possibilities to not have the cdata
        # populated with what we might have expected, so just be smart
        # enough to not raise another KeyError as the name is easily
        # guessable and fallback in all cases to present the real
        # exception to the user
        name = (cdata.get('args') or [None])[0] or cdata['kwargs'].get('name')
        if not name:
            name = low.get('name', low.get('__id__'))

        proc = salt.utils.process.MultiprocessingProcess(
                target=self._call_parallel_target,
                args=(name, cdata, low))
        proc.start()
        ret = {'name': name,
                'result': None,
                'changes': {},
                'comment': 'Started in a separate process',
                'proc': proc}
        return ret
647771
def _multi_call(function, contentkey, *args, **kwargs):
    '''
    
    '''
    ret = function(*args, **kwargs)
    position = ret.get('position')

    while position:
        more = function(*args, position=position, **kwargs)
        ret[contentkey].extend(more[contentkey])
        position = more.get('position')
    return ret.get(contentkey)
638624
def parallel_apply_transformations(x, transforms, black_border_size=0):
  
  transforms = tf.convert_to_tensor(transforms, dtype=tf.float32)
  x = _apply_black_border(x, black_border_size)

  num_transforms = transforms.get_shape().as_list()[0]
  im_shape = x.get_shape().as_list()[1:]

  # Pass a copy of x and a transformation to each iteration of the map_fn
  # callable
  tiled_x = tf.reshape(
      tf.tile(x, [num_transforms, 1, 1, 1]),
      [num_transforms, -1] + im_shape)
  elems = [tiled_x, transforms]
  transformed_ims = tf.map_fn(
      _apply_transformation,
      elems,
      dtype=tf.float32,
      parallel_iterations=1,  # Must be 1 to avoid keras race conditions
  )
  return transformed_ims
223973
def _fill_masked_intensity_resid(intensity_resid, bright_pixel_mask):
        
        filled_intensity = np.zeros((intensity_resid.data.shape))
        nebins = len(intensity_resid.data)
        for i in range(nebins):
            masked = bright_pixel_mask.data[i]
            unmasked = np.invert(masked)
            mean_intensity = intensity_resid.data[i][unmasked].mean()
            filled_intensity[i] = np.where(masked, mean_intensity, intensity_resid.data[i])
        return HpxMap(filled_intensity, intensity_resid.hpx)
71808
def generate_parameters(self, parameter_id):
        
        total_params = self.get_suggestion(random_search=False)
        # avoid generating same parameter with concurrent trials because hyperopt doesn't support parallel mode
        if total_params in self.total_data.values():
            # but it can cause deplicate parameter rarely
            total_params = self.get_suggestion(random_search=True)
        self.total_data[parameter_id] = total_params
        params = _split_index(total_params)
        return params
1009480
def get_parallel_regions_block(batch):
    
    samples = [utils.to_single_data(d) for d in batch]
    regions = _get_parallel_regions(samples[0])
    out = []
    # Currently don't have core information here so aim for about 10 items per partition
    n = 10
    for region_block in tz.partition_all(n, regions):
        out.append({"region_block": ["%s:%s-%s" % (c, s, e) for c, s, e in region_block]})
    return out