Retrieve the detector file for given detector id

        If t0set is given, append the calibration data.
9329
def detx(self, det_id, t0set=None, calibration=None):
        
        url = 'detx/{0}?'.format(det_id)    # '?' since it's ignored if no args
        if t0set is not None:
            url += '&t0set=' + t0set
        if calibration is not None:
            url += '&calibrid=' + calibration

        detx = self._get_content(url)
        return detx
1271824
def get_dataset(self, key, info):
        """"""
        datadict = {
            1000: ['EV_250_Aggr1km_RefSB',
                   'EV_500_Aggr1km_RefSB',
                   'EV_1KM_RefSB',
                   'EV_1KM_Emissive'],
            500: ['EV_250_Aggr500_RefSB',
                  'EV_500_RefSB'],
            250: ['EV_250_RefSB']}

        platform_name = self.metadata['INVENTORYMETADATA']['ASSOCIATEDPLATFORMINSTRUMENTSENSOR'][
            'ASSOCIATEDPLATFORMINSTRUMENTSENSORCONTAINER']['ASSOCIATEDPLATFORMSHORTNAME']['VALUE']

        info.update({'platform_name': 'EOS-' + platform_name})
        info.update({'sensor': 'modis'})

        if self.resolution != key.resolution:
            return

        datasets = datadict[self.resolution]
        for dataset in datasets:
            subdata = self.sd.select(dataset)
            var_attrs = subdata.attributes()
            band_names = var_attrs["band_names"].split(",")

            # get the relative indices of the desired channel
            try:
                index = band_names.index(key.name)
            except ValueError:
                continue
            uncertainty = self.sd.select(dataset + "_Uncert_Indexes")
            array = xr.DataArray(from_sds(subdata, chunks=CHUNK_SIZE)[index, :, :],
                                 dims=['y', 'x']).astype(np.float32)
            valid_range = var_attrs['valid_range']

            # Fill values:
            # Data Value Meaning
            # 65535 Fill Value (includes reflective band data at night mode
            # and completely missing L1A scans)
            # 65534 L1A DN is missing within a scan
            # 65533 Detector is saturated
            # 65532 Cannot compute zero point DN, e.g., SV is saturated
            # 65531 Detector is dead (see comments below)
            # 65530 RSB dn** below the minimum of the scaling range
            # 65529 TEB radiance or RSB dn** exceeds the maximum of the
            # scaling range
            # 65528 Aggregation algorithm failure
            # 65527 Rotation of Earth view Sector from nominal science
            # collection position
            # 65526 Calibration coefficient b1 could not be computed
            # 65525 Subframe is dead
            # 65524 Both sides of the PCLW electronics on simultaneously
            # 65501 - 65523 (reserved for future use)
            # 65500 NAD closed upper limit

            array = array.where(array >= np.float32(valid_range[0]))
            array = array.where(array <= np.float32(valid_range[1]))
            array = array.where(from_sds(uncertainty, chunks=CHUNK_SIZE)[index, :, :] < 15)

            if key.calibration == 'brightness_temperature':
                projectable = calibrate_bt(array, var_attrs, index, key.name)
                info.setdefault('units', 'K')
                info.setdefault('standard_name', 'toa_brightness_temperature')
            elif key.calibration == 'reflectance':
                projectable = calibrate_refl(array, var_attrs, index)
                info.setdefault('units', '%')
                info.setdefault('standard_name',
                                'toa_bidirectional_reflectance')
            elif key.calibration == 'radiance':
                projectable = calibrate_radiance(array, var_attrs, index)
                info.setdefault('units', var_attrs.get('radiance_units'))
                info.setdefault('standard_name',
                                'toa_outgoing_radiance_per_unit_wavelength')
            elif key.calibration == 'counts':
                projectable = calibrate_counts(array, var_attrs, index)
                info.setdefault('units', 'counts')
                info.setdefault('standard_name', 'counts')  # made up
            else:
                raise ValueError("Unknown calibration for "
                                 "key: {}".format(key))
            projectable.attrs = info

            # if ((platform_name == 'Aqua' and key.name in ["6", "27", "36"]) or
            #         (platform_name == 'Terra' and key.name in ["29"])):
            #     height, width = projectable.shape
            #     row_indices = projectable.mask.sum(1) == width
            #     if row_indices.sum() != height:
            #         projectable.mask[row_indices, :] = True

            # Get the orbit number
            # if not satscene.orbit:
            #     mda = self.data.attributes()["CoreMetadata.0"]
            #     orbit_idx = mda.index("ORBITNUMBER")
            #     satscene.orbit = mda[orbit_idx + 111:orbit_idx + 116]

            # Trimming out dead sensor lines (detectors) on terra:
            # (in addition channel 27, 30, 34, 35, and 36 are nosiy)
            # if satscene.satname == "terra":
            #     for band in ["29"]:
            #         if not satscene[band].is_loaded() or satscene[band].data.mask.all():
            #             continue
            #         width = satscene[band].data.shape[1]
            #         height = satscene[band].data.shape[0]
            #         indices = satscene[band].data.mask.sum(1) < width
            #         if indices.sum() == height:
            #             continue
            #         satscene[band] = satscene[band].data[indices, :]
            #         satscene[band].area = geometry.SwathDefinition(
            #             lons=satscene[band].area.lons[indices, :],
            #             lats=satscene[band].area.lats[indices, :])
            return projectable
1147517
def upload_verify(self, file_location, dataset=None):
        """"""

        path = '/api/1.0/upload/verify'
        query = 'doNotGenerateAdvanceReport=true&filePath={}'.format(file_location)
        if dataset:
            query = 'doNotGenerateAdvanceReport=true&filePath={}&datasetId={}'.format(file_location, dataset)

        return self._api_get(definition.UploadVerifyResponse, path, query)
810822
def load_ds_ids_from_config(self):
        """"""
        ids = []
        for dataset in self.datasets.values():
            # xarray doesn't like concatenating attributes that are lists
            # https://github.com/pydata/xarray/issues/2060
            if 'coordinates' in dataset and \
                    isinstance(dataset['coordinates'], list):
                dataset['coordinates'] = tuple(dataset['coordinates'])
            # Build each permutation/product of the dataset
            id_kwargs = []
            for key in DATASET_KEYS:
                val = dataset.get(key)
                if key in ["wavelength", "modifiers"] and isinstance(val,
                                                                     list):
                    # special case: wavelength can be [min, nominal, max]
                    # but is still considered 1 option
                    # it also needs to be a tuple so it can be used in
                    # a dictionary key (DatasetID)
                    id_kwargs.append((tuple(val), ))
                elif key == "modifiers" and val is None:
                    # empty modifiers means no modifiers applied
                    id_kwargs.append((tuple(), ))
                elif isinstance(val, (list, tuple, set)):
                    # this key has multiple choices
                    # (ex. 250 meter, 500 meter, 1000 meter resolutions)
                    id_kwargs.append(val)
                elif isinstance(val, dict):
                    id_kwargs.append(val.keys())
                else:
                    # this key only has one choice so make it a one
                    # item iterable
                    id_kwargs.append((val, ))
            for id_params in itertools.product(*id_kwargs):
                dsid = DatasetID(*id_params)
                ids.append(dsid)

                # create dataset infos specifically for this permutation
                ds_info = dataset.copy()
                for key in DATASET_KEYS:
                    if isinstance(ds_info.get(key), dict):
                        ds_info.update(ds_info[key][getattr(dsid, key)])
                    # this is important for wavelength which was converted
                    # to a tuple
                    ds_info[key] = getattr(dsid, key)
                self.ids[dsid] = ds_info

        return ids
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data
1098051
def load_probe(name):
    """"""
    if op.exists(name):
        # The argument can be either a path to a PRB file.
        path = name
    else:
        # Or the name of a built-in probe.
        curdir = op.realpath(op.dirname(__file__))
        path = op.join(curdir, 'probes/{}.prb'.format(name))
    if not op.exists(path):
        raise IOError("The probe `{}` cannot be found.".format(name))
    return MEA(probe=_read_python(path))
767907
def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
                url=None, resume=True, verbose=1):
    
    warn(DEPRECATION_MSG)
    file_id, md5 = OSF_RESOURCES[dataset_name]
    if url is None:
        url = '{}/{}'.format(OSF_PROJECT_URL, file_id)
    return fetch_file(dataset_name, url, data_dir, dataset_prefix=dataset_prefix,
                      filetype='tar', resume=resume, verbose=verbose, md5sum=md5)
552785
def get_file(self,pattern):
        '''
         
        '''
        flag=[fnmatch.fnmatch(l,pattern) for l in self.filelist]
        id=self.fid_list.compress(flag)
        flag = self.fileid == id
        return flag
267005
def getiddfile(versionid):
    """"""
    vlist = versionid.split('.')
    if len(vlist) == 1:
        vlist = vlist + ['0', '0']
    elif len(vlist) == 2:
        vlist = vlist + ['0']
    ver_str =  '-'.join(vlist)
    eplus_exe, _  = eppy.runner.run_functions.install_paths(ver_str)
    eplusfolder = os.path.dirname(eplus_exe)
    iddfile = '{}/Energy+.idd'.format(eplusfolder, )
    return iddfile
431325
def get_detector(self, id, **kwargs):
        "
        resp = self._get_object_by_name(self._DETECTOR_ENDPOINT_SUFFIX, id,
                                        **kwargs)
        return resp
1286915
def addDataset(self, dataset):
        
        id_ = dataset.getId()
        self._datasetIdMap[id_] = dataset
        self._datasetNameMap[dataset.getLocalId()] = dataset
        self._datasetIds.append(id_)
1185943
def add(self, data, overwrite=False):
        

        if is_srec(data):
            self.add_srec(data, overwrite)
        elif is_ihex(data):
            self.add_ihex(data, overwrite)
        elif is_ti_txt(data):
            self.add_ti_txt(data, overwrite)
        else:
            raise UnsupportedFileFormatError()
806866
def _get_file(self):
        '''
        
        '''
        f = tempfile.NamedTemporaryFile(delete=False)
        self.tmp_files.add(f.name)
        return f
973476
def get_dataset(self, key, info):
        """"""
        logger.debug('Reading dataset {}'.format(key.name))

        # Read data from file and calibrate if necessary
        if 'longitude' in key.name:
            data = self.geo_data['lon']
        elif 'latitude' in key.name:
            data = self.geo_data['lat']
        else:
            tic = datetime.now()
            data = self.calibrate(self.nc['data'].isel(time=0),
                                  calibration=key.calibration,
                                  channel=key.name)
            logger.debug('Calibration time: {}'.format(datetime.now() - tic))

        # Mask space pixels
        data = data.where(self.meta['earth_mask'])

        # Set proper dimension names
        data = data.rename({'xc': 'x', 'yc': 'y'})

        # Update metadata
        self._update_metadata(data, ds_info=info)

        return data
248549
def get_dataset(self, key, info):
        """"""

        if key.name in CHANNEL_NAMES:
            dataset = self.calibrate(key)
        elif key.name in ['longitude', 'latitude']:
            if self.lons is None or self.lats is None:
                self.navigate()
            if key.name == 'longitude':
                dataset = create_xarray(self.lons)
            else:
                dataset = create_xarray(self.lats)
            dataset.attrs = info
        else:  # Get sun-sat angles
            if key.name in ANGLES:
                if isinstance(getattr(self, ANGLES[key.name]), np.ndarray):
                    dataset = create_xarray(getattr(self, ANGLES[key.name]))
                else:
                    dataset = self.get_angles(key.name)
            else:
                raise ValueError("Not a supported sun-sensor viewing angle: %s", key.name)

        dataset.attrs.update({'platform_name': self.platform_name,
                              'sensor': self.sensor})
        dataset.attrs.update(key.to_dict())

        if not self._shape:
            self._shape = dataset.shape

        return dataset
352476
def load_tpf(self):
        '''
        

        '''

        if not self.loaded:
            if self._data is not None:
                data = self._data
            else:
                data = self._mission.GetData(
                         self.ID, season=self.season,
                         cadence=self.cadence,
                         clobber=self.clobber_tpf,
                         aperture_name=self.aperture_name,
                         saturated_aperture_name=self.saturated_aperture_name,
                         max_pixels=self.max_pixels,
                         saturation_tolerance=self.saturation_tolerance,
                         get_hires=self.get_hires,
                         get_nearby=self.get_nearby)
                if data is None:
                    raise Exception("Unable to retrieve target data.")
            self.cadn = data.cadn
            self.time = data.time
            self.model = np.zeros_like(self.time)
            self.fpix = data.fpix
            self.fraw = np.sum(self.fpix, axis=1)
            self.fpix_err = data.fpix_err
            self.fraw_err = np.sqrt(np.sum(self.fpix_err ** 2, axis=1))
            self.nanmask = data.nanmask
            self.badmask = data.badmask
            self.transitmask = np.array([], dtype=int)
            self.outmask = np.array([], dtype=int)
            self.aperture = data.aperture
            self.aperture_name = data.aperture_name
            self.apertures = data.apertures
            self.quality = data.quality
            self.Xpos = data.Xpos
            self.Ypos = data.Ypos
            self.mag = data.mag
            self.pixel_images = data.pixel_images
            self.nearby = data.nearby
            self.hires = data.hires
            self.saturated = data.saturated
            self.meta = data.meta
            self.bkg = data.bkg

            # Update the last breakpoint to the correct value
            self.breakpoints[-1] = len(self.time) - 1

            # Get PLD normalization
            self.get_norm()

            self.loaded = True
158099
def find_experiment_export(app_id):
    

    # Check locally first
    cwd = os.getcwd()
    data_filename = "{}-data.zip".format(app_id)
    path_to_data = os.path.join(cwd, "data", data_filename)
    if os.path.exists(path_to_data):
        try:
            Data(path_to_data)
        except IOError:
            from dallinger import logger

            logger.exception(
                "Error reading local data file {}, checking remote.".format(
                    path_to_data
                )
            )
        else:
            return path_to_data

    # Get remote file instead
    path_to_data = os.path.join(tempfile.mkdtemp(), data_filename)

    buckets = [user_s3_bucket(), dallinger_s3_bucket()]

    for bucket in buckets:
        try:
            bucket.download_file(data_filename, path_to_data)
        except botocore.exceptions.ClientError:
            pass
        else:
            return path_to_data