Calls the interpolation function and then saves the MRMS data to a netCDF file. It will also create 
        separate directories for each variable if they are not already available.
761628
def interpolate_to_netcdf(self, in_lon, in_lat, out_path, date_unit="seconds since 1970-01-01T00:00",
                              interp_type="spline"):
        
        if interp_type == "spline":
            out_data = self.interpolate_grid(in_lon, in_lat)
        else:
            out_data = self.max_neighbor(in_lon, in_lat)
        if not os.access(out_path + self.variable, os.R_OK):
            try:
                os.mkdir(out_path + self.variable)
            except OSError:
                print(out_path + self.variable + " already created")
        out_file = out_path + self.variable + "/" + "{0}_{1}_{2}.nc".format(self.variable,
                                                                            self.start_date.strftime("%Y%m%d-%H:%M"),
                                                                            self.end_date.strftime("%Y%m%d-%H:%M"))
        out_obj = Dataset(out_file, "w")
        out_obj.createDimension("time", out_data.shape[0])
        out_obj.createDimension("y", out_data.shape[1])
        out_obj.createDimension("x", out_data.shape[2])
        data_var = out_obj.createVariable(self.variable, "f4", ("time", "y", "x"), zlib=True, 
                                          fill_value=-9999.0,
                                          least_significant_digit=3)
        data_var[:] = out_data
        data_var.long_name = self.variable
        data_var.coordinates = "latitude longitude"
        if "MESH" in self.variable or "QPE" in self.variable:
            data_var.units = "mm"
        elif "Reflectivity" in self.variable:
            data_var.units = "dBZ"
        elif "Rotation" in self.variable:
            data_var.units = "s-1"
        else:
            data_var.units = ""
        out_lon = out_obj.createVariable("longitude", "f4", ("y", "x"), zlib=True)
        out_lon[:] = in_lon
        out_lon.units = "degrees_east"
        out_lat = out_obj.createVariable("latitude", "f4", ("y", "x"), zlib=True)
        out_lat[:] = in_lat
        out_lat.units = "degrees_north"
        dates = out_obj.createVariable("time", "i8", ("time",), zlib=True)
        dates[:] = np.round(date2num(self.all_dates.to_pydatetime(), date_unit)).astype(np.int64)
        dates.long_name = "Valid date"
        dates.units = date_unit
        out_obj.Conventions="CF-1.6"
        out_obj.close()
        return
517218
def write(self, timeunit, timepoints) -> None:
        """"""
        with netcdf4.Dataset(self.filepath, "w") as ncfile:
            ncfile.Conventions = 'CF-1.6'
            self._insert_timepoints(ncfile, timepoints, timeunit)
            for variable in self.variables.values():
                variable.write(ncfile)
1008788
def _call(self, x, out=None):
        """"""
        # TODO: pass reasonable options on to the interpolator
        def nearest(arg, out=None):
            """Interpolating function with vectorization."""
            if is_valid_input_meshgrid(arg, self.grid.ndim):
                input_type = 'meshgrid'
            else:
                input_type = 'array'

            interpolator = _NearestInterpolator(
                self.grid.coord_vectors, x, variant=self.variant,
                input_type=input_type)

            return interpolator(arg, out=out)

        return self.range.element(nearest, vectorized=True)
257286
def save_model(self, net):
        
        if self.f_params is not None:
            f = self._format_target(net, self.f_params, -1)
            self._save_params(f, net, "f_params", "model parameters")

        if self.f_optimizer is not None:
            f = self._format_target(net, self.f_optimizer, -1)
            self._save_params(f, net, "f_optimizer", "optimizer state")

        if self.f_history is not None:
            f = self.f_history_
            self._save_params(f, net, "f_history", "history")

        if self.f_pickle:
            f_pickle = self._format_target(net, self.f_pickle, -1)
            with open_file_like(f_pickle, 'wb') as f:
                pickle.dump(net, f)
1193536
def save_to_tomodir(self, directory):
        
        self.create_tomodir(directory)

        self.grid.save_elem_file(
            directory + os.sep + 'grid/elem.dat'
        )

        self.grid.save_elec_file(
            directory + os.sep + 'grid/elec.dat'
        )

        # modeling
        if self.configs.configs is not None:
            self.configs.write_crmod_config(
                directory + os.sep + 'config/config.dat'
            )

        if self.assignments['forward_model'] is not None:
            self.parman.save_to_rho_file(
                directory + os.sep + 'rho/rho.dat',
                self.assignments['forward_model'][0],
                self.assignments['forward_model'][1],
            )

        self.crmod_cfg.write_to_file(
            directory + os.sep + 'exe/crmod.cfg'
        )

        if self.assignments['measurements'] is not None:
            self.configs.write_crmod_volt(
                directory + os.sep + 'mod/volt.dat',
                self.assignments['measurements']
            )

        if self.assignments['sensitivities'] is not None:
            self._save_sensitivities(
                directory + os.sep + 'mod/sens',
            )

        if self.assignments['potentials'] is not None:
            self._save_potentials(
                directory + os.sep + 'mod/pot',
            )

        # inversion
        self.crtomo_cfg.write_to_file(
            directory + os.sep + 'exe/crtomo.cfg'
        )

        if self.noise_model is not None:
            self.noise_model.write_crt_noisemod(
                directory + os.sep + 'exe/crt.noisemod'
            )

        if not os.path.isdir(directory + os.sep + 'inv'):
            os.makedirs(directory + os.sep + 'inv')
40407
def load_data(self):
        
        data = []
        valid_dates = []
        mrms_files = np.array(sorted(os.listdir(self.path + self.variable + "/")))
        mrms_file_dates = np.array([m_file.split("_")[-2].split("-")[0]
            for m_file in mrms_files])
        old_mrms_file = None
        file_obj = None
        for t in range(self.all_dates.shape[0]):
            file_index = np.where(mrms_file_dates == self.all_dates[t].strftime("%Y%m%d"))[0]
            if len(file_index) > 0:
                mrms_file = mrms_files[file_index][0]
                if mrms_file is not None:
                    if file_obj is not None:
                        file_obj.close()
                    file_obj = Dataset(self.path + self.variable + "/" + mrms_file)
                    #old_mrms_file = mrms_file
                    
                    if "time" in file_obj.variables.keys():
                        time_var = "time"
                    else:
                        time_var = "date"
                    file_valid_dates = pd.DatetimeIndex(num2date(file_obj.variables[time_var][:],
                                                                 file_obj.variables[time_var].units))
                else:
                    file_valid_dates = pd.DatetimeIndex([])
                time_index = np.where(file_valid_dates.values == self.all_dates.values[t])[0]
                if len(time_index) > 0:
                    data.append(file_obj.variables[self.variable][time_index[0]])
                    valid_dates.append(self.all_dates[t])
        if file_obj is not None:
            file_obj.close()
        self.data = np.array(data)
        self.data[self.data < 0] = 0
        self.data[self.data > 150] = 150
        self.valid_dates = pd.DatetimeIndex(valid_dates)
217293
def save(fname, d, link_copy=True,raiseError=False):
    
    # make sure the object is dict (recursively) this allows reading it
    # without the DataStorage module
    fname = pathlib.Path(fname)
    d = toDict(d, recursive=True)
    d['filename'] = str(fname)
    extension = fname.suffix
    log.info("Saving storage file %s" % fname)
    try:
        if extension == ".npz":
            return dictToNpz(fname, d)
        elif extension == ".h5":
            return dictToH5(fname, d, link_copy=link_copy)
        elif extension == ".npy":
            return dictToNpy(fname, d)
        else:
            raise ValueError(
                "Extension must be h5, npy or npz, it was %s" % extension)
    except Exception as e:
        log.exception("Could not save %s" % fname)
        if raiseError: raise  e
574053
def save_volume_files(note, error, registrations, subject,
                      no_vol_export, volume_format, volume_path,
                      angle_tag, eccen_tag, label_tag, radius_tag):
    '''
    
    '''
    if no_vol_export: return {'volume_files': ()}
    volume_format = volume_format.lower()
    # make an exporter for properties:
    if volume_format in ['mgh', 'mgz', 'auto', 'automatic', 'default']:
        volume_format = 'mgh' if volume_format == 'mgh' else 'mgz'
        def export(flnm, d):
            flnm = flnm + '.' + volume_format
            dt = np.int32 if np.issubdtype(d.dtype, np.dtype(int).type) else np.float32
            img = fsmgh.MGHImage(np.asarray(d, dtype=dt), subject.voxel_to_native_matrix)
            img.to_filename(flnm)
            return flnm
    elif volume_format in ['nifti', 'nii', 'niigz', 'nii.gz']:
        volume_format = 'nii' if volume_format == 'nii' else 'nii.gz'
        def export(flnm, p):
            flnm = flnm + '.' + volume_format
            dt = np.int32 if np.issubdtype(p.dtype, np.dtype(int).type) else np.float32
            img = nib.Nifti1Image(np.asarray(p, dtype=dt), subject.voxel_to_native_matrix)
            img.to_filename(flnm)
            return flnm
    else:
        error('Could not understand volume file-format %s' % volume_format)
    path = volume_path if volume_path else os.path.join(subject.path, 'mri')
    files = []
    note('Extracting predicted meshes for volume export...')
    hemis = [registrations[h]['predicted_mesh'] if h in registrations else None
             for h in ['lh', 'rh']]
    for (pname,tag) in zip(['polar_angle', 'eccentricity', 'visual_area', 'radius'],
                           [angle_tag, eccen_tag, label_tag, radius_tag]):
        # we have to make the volume first...
        dat = tuple([None if h is None else h.prop(pname) for h in hemis])
        (mtd,dt) = ('nearest',np.int32) if pname == 'visual_area' else ('linear',np.float32)
        note('Constructing %s image...' % pname)
        img = subject.cortex_to_image(dat, method=mtd, dtype=dt)
        flnm = export(os.path.join(path, tag), img)
        files.append(flnm)
    return {'volume_files': tuple(files)}
1117685
def load_local_dataset(self, ds_str):
        '''
        
        '''
        if cdl.is_cdl(ds_str):
            ds_str = self.generate_dataset(ds_str)

        if netcdf.is_netcdf(ds_str):
            return MemoizedDataset(ds_str)

        # Assume this is just a Generic File if it exists
        if os.path.isfile(ds_str):
            return GenericFile(ds_str)

        raise ValueError("File is an unknown format")
130963
def write_NetCDF_georeference(origin, outfile):
    

    geo_ref = ensure_geo_reference(origin)
    geo_ref.write_NetCDF(outfile)
    return geo_ref
1317416
def interpolate_linear(self, lons, lats, data):
        
        return self.interpolate(lons, lats, data, order=1)
551699
def interpolate(self, doy, depth, lat, lon, var):
        
        subset, dims = self.crop(doy, depth, lat, lon, var)

        # Subset contains everything requested. No need to interpolate.
        if np.all([d in dims['time'] for d in doy]) & \
                np.all([z in dims['depth'] for z in depth]) & \
                np.all([y in dims['lat'] for y in lat]) & \
                np.all([x in dims['lon'] for x in lon]):
                    dn = np.nonzero([d in doy for d in dims['time']])[0]
                    zn = np.nonzero([z in depth for z in dims['depth']])[0]
                    yn = np.nonzero([y in lat for y in dims['lat']])[0]
                    xn = np.nonzero([x in lon for x in dims['lon']])[0]
                    output = {}
                    for v in subset:
                        # output[v] = subset[v][dn, zn, yn, xn]
                        # Seriously that this is the way to do it?!!??
                        output[v] = subset[v][:, :, :, xn][:, :, yn][:, zn][dn]
                    return output

        output = {}
        for v in var:
            output[v] = ma.masked_all(
                    (doy.size, depth.size, lat.size, lon.size),
                    dtype=subset[v].dtype)

            # These interpolators don't understand Masked Arrays, but do NaN
            if subset[v].dtype in ['int32']:
                subset[v] = subset[v].astype('f')
            subset[v][ma.getmaskarray(subset[v])] = np.nan
            subset[v] = subset[v].data

        # First linear interpolate on time.
        if not (doy == dims['time']).all():
            for v in subset.keys():
                f = interp1d(dims['time'], subset[v], axis=0)
                subset[v] = f(doy)
            dims['time'] = np.atleast_1d(doy)

        if not (np.all(lat == dims['lat']) and np.all(lon == dims['lon'])):
            # Lat x Lon target coordinates are the same for all time and depth.
            points_out = []
            for latn in lat:
                for lonn in lon:
                    points_out.append([latn, lonn])
            points_out = np.array(points_out)

            # Interpolate on X/Y plane
            for v in subset:
                tmp = np.nan * np.ones(
                        (doy.size, dims['depth'].size, lat.size, lon.size),
                        dtype=subset[v].dtype)
                for nt in range(doy.size):
                    for nz in range(dims['depth'].size):
                        data = subset[v][nt, nz]
                        # The valid data
                        idx = np.nonzero(~np.isnan(data))
                        if idx[0].size > 0:
                            points = np.array([
                                dims['lat'][idx[0]], dims['lon'][idx[1]]]).T
                            values = data[idx]
                            # Interpolate along the dimensions that have more than
                            #   one position, otherwise it means that the output
                            #   is exactly on that coordinate.
                            #ind = np.array([np.unique(points[:, i]).size > 1
                            #    for i in range(points.shape[1])])
                            #assert ind.any()

                            try:
                                values_out = griddata(
                                    #np.atleast_1d(np.squeeze(points[:, ind])),
                                    np.atleast_1d(np.squeeze(points)),
                                    values,
                                    #np.atleast_1d(np.squeeze(points_out[:, ind])))
                                    np.atleast_1d(np.squeeze(points_out)))
                            except:
                                values_out = []
                                for p in points_out:
                                    try:
                                        values_out.append(griddata(
                                            np.atleast_1d(np.squeeze(points)),
                                            values,
                                            np.atleast_1d(np.squeeze(
                                                p))))
                                    except:
                                        values_out.append(np.nan)
                                values_out = np.array(values_out)

                            # Remap the interpolated value back into a 4D array
                            idx = np.isfinite(values_out)
                            for [y, x], out in zip(
                                    points_out[idx], values_out[idx]):
                                tmp[nt, nz, y==lat, x==lon] = out
                subset[v] = tmp

        # Interpolate on z
        same_depth = (np.shape(depth) == dims['depth'].shape) and \
                np.allclose(depth, dims['depth'])
        if not same_depth:
            for v in list(subset.keys()):
                try:
                    f = interp1d(dims['depth'], subset[v], axis=1, bounds_error=False)
                    # interp1d does not handle Masked Arrays
                    subset[v] = f(np.array(depth))
                except:
                    print("Fail to interpolate '%s' in depth" % v)
                    del(subset[v])

        for v in subset:
            if output[v].dtype in ['int32']:
                subset[v] = np.round(subset[v])
            output[v][:] = ma.fix_invalid(subset[v][:])

        return output
1229312
def export(cls, folder, particles, datetimes, summary, **kwargs):
        
        time_units = 'seconds since 1990-01-01 00:00:00'

        # Create netcdf file, overwrite existing
        if not os.path.exists(folder):
            os.makedirs(folder)

        filepath = os.path.join(folder, 'trajectories.nc')
        nc = netCDF4.Dataset(filepath, 'w')
        # Create netcdf dimensions
        nc.createDimension('time', None)
        nc.createDimension('particle', None)

        fillvalue = -9999.9

        # Create netcdf variables
        time = nc.createVariable('time', 'i', ('time',))
        part = nc.createVariable('particle', 'i', ('particle',))
        depth = nc.createVariable('depth', 'f', ('time', 'particle'))
        lat = nc.createVariable('lat', 'f', ('time', 'particle'), fill_value=fillvalue)
        lon = nc.createVariable('lon', 'f', ('time', 'particle'), fill_value=fillvalue)
        salt = nc.createVariable('salt', 'f', ('time', 'particle'), fill_value=fillvalue)
        temp = nc.createVariable('temp', 'f', ('time', 'particle'), fill_value=fillvalue)
        u = nc.createVariable('u', 'f', ('time', 'particle'), fill_value=fillvalue)
        v = nc.createVariable('v', 'f', ('time', 'particle'), fill_value=fillvalue)
        w = nc.createVariable('w', 'f', ('time', 'particle'), fill_value=fillvalue)
        settled = nc.createVariable('settled', 'f', ('time', 'particle'), fill_value=fillvalue)
        dead = nc.createVariable('dead', 'f', ('time', 'particle'), fill_value=fillvalue)
        halted = nc.createVariable('halted', 'f', ('time', 'particle'), fill_value=fillvalue)

        # Loop through locations in each particle,
        # add to netcdf file
        for j, particle in enumerate(particles):
            part[j] = particle.uid
            i = 0

            normalized_locations = particle.normalized_locations(datetimes)
            normalized_temps = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.temps]
            normalized_salts = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.salts]
            normalized_u = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.u_vectors]
            normalized_v = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.v_vectors]
            normalized_w = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.w_vectors]
            normalized_settled = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.settles]
            normalized_dead = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.deads]
            normalized_halted = [x if x is not None and not math.isnan(x) else fillvalue for x in particle.halts]

            if len(normalized_locations) != len(normalized_temps):
                logger.info("No temperature being added to netcdf.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_temps = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_salts):
                logger.info("No salinity being added to netcdf.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_salts = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_u):
                logger.info("No U being added to netcdf.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_u = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_v):
                logger.info("No V being added to netcdf.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_v = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_w):
                logger.info("No W being added to netcdf.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_w = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_settled):
                logger.info("No Settled being added to shapefile.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_settled = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_dead):
                logger.info("No Dead being added to shapefile.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_dead = [fillvalue] * len(normalized_locations)

            if len(normalized_locations) != len(normalized_halted):
                logger.info("No Halted being added to shapefile.")
                # Create list of 'fillvalue' equal to the length of locations
                normalized_halted = [fillvalue] * len(normalized_locations)

            for loc, _temp, _salt, _u, _v, _w, _settled, _dead, _halted in zip(normalized_locations, normalized_temps, normalized_salts, normalized_u, normalized_v, normalized_w, normalized_settled, normalized_dead, normalized_halted):

                if j == 0:
                    time[i] = int(round(netCDF4.date2num(loc.time, time_units)))
                depth[i, j] = loc.depth
                lat[i, j] = loc.latitude
                lon[i, j] = loc.longitude
                salt[i, j] = _salt
                temp[i, j] = _temp
                u[i, j] = _u
                v[i, j] = _v
                w[i, j] = _w
                settled[i, j] = _settled
                dead[i, j] = _dead
                halted[i, j] = _halted
                i += 1

        # Variable attributes
        depth.coordinates = "time particle lat lon"
        depth.standard_name = "depth_below_sea_surface"
        depth.units = "m"
        depth.POSITIVE = "up"
        depth.positive = "up"

        salt.coordinates = "time particle lat lon"
        salt.standard_name = "sea_water_salinity"
        salt.units = "psu"

        temp.coordinates = "time particle lat lon"
        temp.standard_name = "sea_water_temperature"
        temp.units = "degrees_C"

        u.coordinates = "time particle lat lon"
        u.standard_name = "eastward_sea_water_velocity"
        u.units = "m/s"

        v.coordinates = "time particle lat lon"
        v.standard_name = "northward_sea_water_velocity"
        v.units = "m/s"

        w.coordinates = "time particle lat lon"
        w.standard_name = "upward_sea_water_velocity"
        w.units = "m/s"

        settled.coordinates = "time particle lat lon"
        settled.description = "Is the particle settled"
        settled.standard_name = "particle_settled"

        dead.coordinates = "time particle lat lon"
        dead.description = "Is the particle dead"
        dead.standard_name = "particle_dead"

        halted.coordinates = "time particle lat lon"
        halted.description = "Is the particle prevented from being forced by currents"
        halted.standard_name = "particle_halted"

        time.units = time_units
        time.standard_name = "time"

        lat.units = "degrees_north"

        lon.units = "degrees_east"

        part.cf_role = "trajectory_id"

        # Global attributes
        nc.featureType = "trajectory"
        nc.summary = str(summary)
        for key in kwargs:
            nc.__setattr__(key, kwargs.get(key))

        nc.sync()
        nc.close()
1226304
def interpolate(p, axis_values, pixelgrid, order=1, mode='constant', cval=0.0):
    
    # convert requested parameter combination into a coordinate
    p_ = np.array([np.searchsorted(av_,val) for av_, val in zip(axis_values,p)])
    lowervals_stepsize = np.array([[av_[p__-1], av_[p__]-av_[p__-1]] \
                         for av_, p__ in zip(axis_values,p_)])
    p_coord = (p-lowervals_stepsize[:,0])/lowervals_stepsize[:,1] + p_-1
    
    # interpolate
    if order > 1:
        prefilter = False
    else:
        prefilter = False

    return [ndimage.map_coordinates(pixelgrid[...,i],p_coord, order=order,
                                    prefilter=prefilter, mode=mode, cval=cval) \
                for i in range(np.shape(pixelgrid)[-1])]
1115297
def interpolate(self):
        
        nans, x = utils.nan_idx(self)
        self[nans] = np.interp(x(nans), x(~nans), self[~nans])
        return self
51584
def func_interpolate_na(interpolator, x, y, **kwargs):
    ''''''
    # it would be nice if this wasn't necessary, works around:
    # "ValueError: assignment destination is read-only" in assignment below
    out = y.copy()

    nans = pd.isnull(y)
    nonans = ~nans

    # fast track for no-nans and all-nans cases
    n_nans = nans.sum()
    if n_nans == 0 or n_nans == len(y):
        return y

    f = interpolator(x[nonans], y[nonans], **kwargs)
    out[nans] = f(x[nans])
    return out
59601
def _save_potentials(self, directory):
        
        print('saving potentials')
        digits = int(np.ceil(np.log10(self.configs.configs.shape[0])))
        for i in range(0, self.configs.configs.shape[0]):
            pot_data = self.get_potential(i)
            filename_raw = 'pot{0:0' + '{0}'.format(digits) + '}.dat'
            filename = directory + os.sep + filename_raw.format(i + 1)

            nodes = self.grid.nodes['sorted'][:, 1:3]
            all_data = np.hstack((
                nodes,
                pot_data[0][:, np.newaxis],
                pot_data[1][:, np.newaxis],
            ))
            with open(filename, 'wb') as fid:
                np.savetxt(fid, all_data)