Scipy.ndimage median filter

    Does not properly handle NaN
517870
def median_fltr(dem, fsize=7, origmask=False):
    
    print("Applying median filter with size %s" % fsize)
    from scipy.ndimage.filters import median_filter
    dem_filt_med = median_filter(dem.filled(np.nan), fsize)
    #Now mask all nans
    out = np.ma.fix_invalid(dem_filt_med, copy=False, fill_value=dem.fill_value)
    if origmask:
        out = np.ma.array(out, mask=dem.mask, fill_value=dem.fill_value)
    out.set_fill_value(dem.fill_value)
    return out
343601
def subtract_column_median(df, prefix='Intensity '):
    
    df = df.copy()

    df.replace([np.inf, -np.inf], np.nan, inplace=True)

    mask = [l.startswith(prefix) for l in df.columns.values]
    df.iloc[:, mask] = df.iloc[:, mask] - df.iloc[:, mask].median(axis=0)

    return df
874440
def first(values, axis, skipna=None):
    
    if (skipna or skipna is None) and values.dtype.kind not in 'iSU':
        # only bother for dtypes that can hold NaN
        _fail_on_dask_array_input_skipna(values)
        return nanfirst(values, axis)
    return take(values, 0, axis=axis)
904438
def NS(s, o):
    
    # s,o = filter_nan(s,o)
    return 1 - np.sum((s-o)**2)/np.sum((o-np.mean(o))**2)
914731
def runningstd(t, data, width):
    
    ne = len(t) - width
    t_new = np.zeros(ne)
    std_r = np.zeros(ne)
    for i in range(ne):
        t_new[i] = np.mean(t[i:i+width+1])
        std_r[i] = scipy.stats.nanstd(data[i:i+width+1])
    return t_new, std_r
893881
def nan_pixels(self):
        
        nan_px = np.where(np.isnan(np.sum(self.raw_data, axis=2)))
        nan_px = np.c_[nan_px[0], nan_px[1]]
        return nan_px
413888
def maskedFilter(arr, mask, ksize=30, fill_mask=True,
                 fn='median'):
    '''
    
    '''

    if fill_mask:
        mask1 = mask
        out = arr
    else:
        mask1 = ~mask
        out = np.full_like(arr, fill_value=np.nan)
    mask2 = ~mask

    if fn == 'mean':
        _calcMean(arr, mask1, mask2, out, ksize // 2)
    else:
        buff = np.empty(shape=(ksize * ksize), dtype=arr.dtype)
        _calcMedian(arr, mask1, mask2, out, ksize // 2, buff)
    return out
250833
def _extract_median(image, mask = slice(None), size = 1, voxelspacing = None):
    
    # set voxel spacing
    if voxelspacing is None:
        voxelspacing = [1.] * image.ndim
        
    # determine structure element size in voxel units
    size = _create_structure_array(size, voxelspacing)
        
    return _extract_intensities(median_filter(image, size), mask)
981445
def insert_size_stats(dists):
    
    med = numpy.median(dists)
    filter_dists = list(filter(lambda x: x < med + 10 * med, dists))
    median = numpy.median(filter_dists)
    return {"mean": float(numpy.mean(filter_dists)), "std": float(numpy.std(filter_dists)),
            "median": float(median),
            "mad": float(numpy.median([abs(x - median) for x in filter_dists]))}
1151074
def _get_mask(X, value_to_mask):
    """"""
    if is_scalar_nan(value_to_mask):
        if X.dtype.kind == "f":
            return np.isnan(X)
        elif X.dtype.kind in ("i", "u"):
            # can't have NaNs in integer array.
            return np.zeros(X.shape, dtype=bool)
        else:
            # np.isnan does not work on object dtypes.
            return _object_dtype_isnan(X)
    else:
        # X == value_to_mask with object dytpes does not always perform
        # element-wise for old versions of numpy
        return np.equal(X, value_to_mask)
31820
def filter_nan(s, o):
    
    data = np.array([s.flatten(), o.flatten()])
    data = np.transpose(data)
    data = data[~np.isnan(data).any(1)]
    return data[:, 0], data[:, 1]
1341536
def _replace_nan(a, val):
    
    mask = isnull(a)
    return where_method(val, mask, a), mask
1309132
def get_MD_VDS(tail_check_max, vds):
    
    if tail_check_max == 0 or numpy.isnan(tail_check_max):
        return float('nan')
    MD_VDS = (old_div(tail_check_max, vds)) * 100
    return MD_VDS
781757
def _norm_kwargs(self, element, ranges, opts, vdim, values=None, prefix=''):
        
        dim_name = dim_range_key(vdim)
        if values is None:
            if isinstance(vdim, dim):
                values = vdim.apply(element, flat=True)
            else:
                expanded = not (
                    isinstance(element, Dataset) and
                    element.interface.multi and
                    (getattr(element, 'level', None) is not None or
                     element.interface.isscalar(element, vdim.name))
                )
                values = np.asarray(element.dimension_values(vdim, expanded=expanded))

        # Store dimension being colormapped for colorbars
        if prefix+'color_dim' not in self.handles:
            self.handles[prefix+'color_dim'] = vdim

        clim = opts.pop(prefix+'clims', None)

        # check if there's an actual value (not np.nan)
        if clim is None and util.isfinite(self.clim).all():
            clim = self.clim

        if clim is None:
            if not len(values):
                clim = (0, 0)
                categorical = False
            elif values.dtype.kind in 'uif':
                if dim_name in ranges:
                    clim = ranges[dim_name]['combined']
                elif isinstance(vdim, dim):
                    if values.dtype.kind == 'M':
                        clim = values.min(), values.max()
                    elif len(values) == 0:
                        clim = np.NaN, np.NaN
                    else:
                        try:
                            with warnings.catch_warnings():
                                warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered')
                                clim = (np.nanmin(values), np.nanmax(values))
                        except:
                            clim = np.NaN, np.NaN
                else:
                    clim = element.range(vdim)
                if self.logz:
                    # Lower clim must be >0 when logz=True
                    # Choose the maximum between the lowest non-zero value
                    # and the overall range
                    if clim[0] == 0:
                        clim = (values[values!=0].min(), clim[1])
                if self.symmetric:
                    clim = -np.abs(clim).max(), np.abs(clim).max()
                categorical = False
            else:
                range_key = dim_range_key(vdim)
                if range_key in ranges and 'factors' in ranges[range_key]:
                    factors = ranges[range_key]['factors']
                else:
                    factors = util.unique_array(values)
                clim = (0, len(factors)-1)
                categorical = True
        else:
            categorical = values.dtype.kind not in 'uif'

        if self.logz:
            if self.symmetric:
                norm = mpl_colors.SymLogNorm(vmin=clim[0], vmax=clim[1],
                                             linthresh=clim[1]/np.e)
            else:
                norm = mpl_colors.LogNorm(vmin=clim[0], vmax=clim[1])
            opts[prefix+'norm'] = norm
        opts[prefix+'vmin'] = clim[0]
        opts[prefix+'vmax'] = clim[1]

        cmap = opts.get(prefix+'cmap', opts.get('cmap', 'viridis'))
        if values.dtype.kind not in 'OSUM':
            ncolors = None
            if isinstance(self.color_levels, int):
                ncolors = self.color_levels
            elif isinstance(self.color_levels, list):
                ncolors = len(self.color_levels) - 1
                if isinstance(cmap, list) and len(cmap) != ncolors:
                    raise ValueError('The number of colors in the colormap '
                                     'must match the intervals defined in the '
                                     'color_levels, expected %d colors found %d.'
                                     % (ncolors, len(cmap)))
            try:
                el_min, el_max = np.nanmin(values), np.nanmax(values)
            except ValueError:
                el_min, el_max = -np.inf, np.inf
        else:
            ncolors = clim[-1]+1
            el_min, el_max = -np.inf, np.inf
        vmin = -np.inf if opts[prefix+'vmin'] is None else opts[prefix+'vmin']
        vmax = np.inf if opts[prefix+'vmax'] is None else opts[prefix+'vmax']
        if el_min < vmin and el_max > vmax:
            self._cbar_extend = 'both'
        elif el_min < vmin:
            self._cbar_extend = 'min'
        elif el_max > vmax:
            self._cbar_extend = 'max'

        # Define special out-of-range colors on colormap
        colors = {}
        for k, val in self.clipping_colors.items():
            if val == 'transparent':
                colors[k] = {'color': 'w', 'alpha': 0}
            elif isinstance(val, tuple):
                colors[k] = {'color': val[:3],
                             'alpha': val[3] if len(val) > 3 else 1}
            elif isinstance(val, util.basestring):
                color = val
                alpha = 1
                if color.startswith('#') and len(color) == 9:
                    alpha = int(color[-2:], 16)/255.
                    color = color[:-2]
                colors[k] = {'color': color, 'alpha': alpha}

        if not isinstance(cmap, mpl_colors.Colormap):
            if isinstance(cmap, dict):
                factors = util.unique_array(values)
                palette = [cmap.get(f, colors.get('NaN', {'color': self._default_nan})['color'])
                           for f in factors]
            else:
                palette = process_cmap(cmap, ncolors, categorical=categorical)
                if isinstance(self.color_levels, list):
                    palette, (vmin, vmax) = color_intervals(palette, self.color_levels, clip=(vmin, vmax))
            cmap = mpl_colors.ListedColormap(palette)
        if 'max' in colors: cmap.set_over(**colors['max'])
        if 'min' in colors: cmap.set_under(**colors['min'])
        if 'NaN' in colors: cmap.set_bad(**colors['NaN'])
        opts[prefix+'cmap'] = cmap
549673
def median_high(data):
    
    data = sorted(data)
    n = len(data)
    if n == 0:
        raise StatisticsError("no median for empty data")
    return data[n // 2]
332273
def df_filter_row_sum(df, threshold, take_abs=True):
  '''  '''

  from copy import deepcopy
  from .__init__ import Network
  net = Network()

  if take_abs is True:
    df_copy = deepcopy(df['mat'].abs())
  else:
    df_copy = deepcopy(df['mat'])

  ini_rows = df_copy.index.values.tolist()
  df_copy = df_copy.transpose()
  tmp_sum = df_copy.sum(axis=0)
  tmp_sum = tmp_sum.abs()
  tmp_sum.sort_values(inplace=True, ascending=False)

  tmp_sum = tmp_sum[tmp_sum > threshold]
  keep_rows = sorted(tmp_sum.index.values.tolist())

  if len(keep_rows) < len(ini_rows):
    df['mat'] = grab_df_subset(df['mat'], keep_rows=keep_rows)

    if 'mat_up' in df:
      df['mat_up'] = grab_df_subset(df['mat_up'], keep_rows=keep_rows)
      df['mat_dn'] = grab_df_subset(df['mat_dn'], keep_rows=keep_rows)

    if 'mat_orig' in df:
      df['mat_orig'] = grab_df_subset(df['mat_orig'], keep_rows=keep_rows)

  return df
301735
def medial_wall_to_nan(in_file, subjects_dir, target_subject, newpath=None):
    
    import nibabel as nb
    import numpy as np
    import os

    fn = os.path.basename(in_file)
    if not target_subject.startswith('fs'):
        return in_file

    cortex = nb.freesurfer.read_label(os.path.join(
        subjects_dir, target_subject, 'label', '{}.cortex.label'.format(fn[:2])))
    func = nb.load(in_file)
    medial = np.delete(np.arange(len(func.darrays[0].data)), cortex)
    for darray in func.darrays:
        darray.data[medial] = np.nan

    out_file = os.path.join(newpath or os.getcwd(), fn)
    func.to_filename(out_file)
    return out_file